2024-11-13 18:23:30,314 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-13 18:23:30,345 main DEBUG Took 0.019181 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 18:23:30,346 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 18:23:30,346 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 18:23:30,348 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 18:23:30,400 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,421 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 18:23:30,455 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,492 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,494 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,496 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,497 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,498 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,506 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,508 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,509 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,510 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,511 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,512 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,512 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,513 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,515 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,515 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,516 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,517 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,518 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:23:30,532 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,533 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 18:23:30,542 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:23:30,544 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 18:23:30,549 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 18:23:30,559 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 18:23:30,580 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 18:23:30,581 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 18:23:30,600 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 18:23:30,605 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 18:23:30,628 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 18:23:30,628 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 18:23:30,629 main DEBUG createAppenders(={Console}) 2024-11-13 18:23:30,632 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-13 18:23:30,635 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-13 18:23:30,635 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-13 18:23:30,636 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 18:23:30,637 main DEBUG OutputStream closed 2024-11-13 18:23:30,637 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 18:23:30,638 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 18:23:30,638 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4b0d79fc OK 2024-11-13 18:23:30,855 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 18:23:30,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 18:23:30,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 18:23:30,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 18:23:30,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 18:23:30,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 18:23:30,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 18:23:30,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 18:23:30,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 18:23:30,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 18:23:30,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 18:23:30,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 18:23:30,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 18:23:30,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 18:23:30,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 18:23:30,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 18:23:30,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 18:23:30,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 18:23:30,927 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 18:23:30,928 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@68b32e3e) with optional ClassLoader: null 2024-11-13 18:23:30,932 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 18:23:30,933 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@68b32e3e] started OK. 2024-11-13T18:23:31,566 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7 2024-11-13 18:23:31,571 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 18:23:31,571 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T18:23:31,587 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-13T18:23:31,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T18:23:31,696 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/cluster_289219f1-227d-9b3f-386e-b9c909da3090, deleteOnExit=true 2024-11-13T18:23:31,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T18:23:31,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/test.cache.data in system properties and HBase conf 2024-11-13T18:23:31,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T18:23:31,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/hadoop.log.dir in system properties and HBase conf 2024-11-13T18:23:31,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T18:23:31,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T18:23:31,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T18:23:31,885 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T18:23:32,049 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:23:32,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T18:23:32,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T18:23:32,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T18:23:32,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T18:23:32,064 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T18:23:32,065 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T18:23:32,071 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T18:23:32,072 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T18:23:32,072 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T18:23:32,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/nfs.dump.dir in system properties and HBase conf 2024-11-13T18:23:32,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/java.io.tmpdir in system properties and HBase conf 2024-11-13T18:23:32,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T18:23:32,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T18:23:32,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T18:23:33,853 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T18:23:34,094 INFO [Time-limited test {}] log.Log(170): Logging initialized @5279ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T18:23:34,242 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:23:34,375 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:23:34,478 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:23:34,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:23:34,482 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T18:23:34,504 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:23:34,511 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:23:34,513 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:23:34,917 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f405bdd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/java.io.tmpdir/jetty-localhost-38007-hadoop-hdfs-3_4_1-tests_jar-_-any-12956717346485324571/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:23:34,934 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:38007} 2024-11-13T18:23:34,935 INFO [Time-limited test {}] server.Server(415): Started @6124ms 2024-11-13T18:23:36,258 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:23:36,279 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:23:36,299 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:23:36,299 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:23:36,300 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T18:23:36,303 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@350067e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:23:36,305 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24457f8e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:23:36,611 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1e531827{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/java.io.tmpdir/jetty-localhost-39827-hadoop-hdfs-3_4_1-tests_jar-_-any-9565070045726236511/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:23:36,621 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4ea205b6{HTTP/1.1, (http/1.1)}{localhost:39827} 2024-11-13T18:23:36,622 INFO [Time-limited test {}] server.Server(415): Started @7811ms 2024-11-13T18:23:36,731 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T18:23:39,426 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/cluster_289219f1-227d-9b3f-386e-b9c909da3090/data/data2/current/BP-1851317081-172.17.0.3-1731522213332/current, will proceed with Du for space computation calculation, 2024-11-13T18:23:39,426 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/cluster_289219f1-227d-9b3f-386e-b9c909da3090/data/data1/current/BP-1851317081-172.17.0.3-1731522213332/current, will proceed with Du for space computation calculation, 2024-11-13T18:23:39,524 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T18:23:39,657 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd4595314247f816 with lease ID 0x3cc3993d007392d4: Processing first storage report for DS-cab5a945-efa3-4827-b04b-1ecd7edf5cb9 from datanode DatanodeRegistration(127.0.0.1:46535, datanodeUuid=5776e202-595b-4c76-93e7-e9f26f4804af, infoPort=40951, infoSecurePort=0, ipcPort=39917, storageInfo=lv=-57;cid=testClusterID;nsid=416548708;c=1731522213332) 2024-11-13T18:23:39,659 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd4595314247f816 with lease ID 0x3cc3993d007392d4: from storage DS-cab5a945-efa3-4827-b04b-1ecd7edf5cb9 node DatanodeRegistration(127.0.0.1:46535, datanodeUuid=5776e202-595b-4c76-93e7-e9f26f4804af, infoPort=40951, infoSecurePort=0, ipcPort=39917, storageInfo=lv=-57;cid=testClusterID;nsid=416548708;c=1731522213332), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-13T18:23:39,660 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd4595314247f816 with lease ID 0x3cc3993d007392d4: Processing first storage report for DS-8b87768d-51ab-495f-a845-9c6c0e2ea43e from datanode DatanodeRegistration(127.0.0.1:46535, datanodeUuid=5776e202-595b-4c76-93e7-e9f26f4804af, infoPort=40951, infoSecurePort=0, ipcPort=39917, storageInfo=lv=-57;cid=testClusterID;nsid=416548708;c=1731522213332) 2024-11-13T18:23:39,660 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd4595314247f816 with lease ID 0x3cc3993d007392d4: from storage DS-8b87768d-51ab-495f-a845-9c6c0e2ea43e node DatanodeRegistration(127.0.0.1:46535, datanodeUuid=5776e202-595b-4c76-93e7-e9f26f4804af, infoPort=40951, infoSecurePort=0, ipcPort=39917, storageInfo=lv=-57;cid=testClusterID;nsid=416548708;c=1731522213332), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T18:23:39,691 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7 2024-11-13T18:23:40,018 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/cluster_289219f1-227d-9b3f-386e-b9c909da3090/zookeeper_0, clientPort=63431, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/cluster_289219f1-227d-9b3f-386e-b9c909da3090/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/cluster_289219f1-227d-9b3f-386e-b9c909da3090/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T18:23:40,039 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63431 2024-11-13T18:23:40,055 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:23:40,059 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:23:40,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741825_1001 (size=7) 2024-11-13T18:23:40,998 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b with version=8 2024-11-13T18:23:40,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/hbase-staging 2024-11-13T18:23:41,288 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T18:23:42,441 INFO [Time-limited test {}] client.ConnectionUtils(128): master/62ae331128c9:0 server-side Connection retries=45 2024-11-13T18:23:42,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:23:42,483 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:23:42,517 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:23:42,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:23:42,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:23:43,239 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T18:23:43,512 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T18:23:43,569 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T18:23:43,578 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:23:43,723 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5042 (auto-detected) 2024-11-13T18:23:43,725 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T18:23:43,900 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45483 2024-11-13T18:23:44,035 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45483 connecting to ZooKeeper ensemble=127.0.0.1:63431 2024-11-13T18:23:44,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:454830x0, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:23:44,361 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45483-0x101369e15260000 connected 2024-11-13T18:23:44,629 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:23:44,632 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:23:44,681 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:23:44,703 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b, hbase.cluster.distributed=false 2024-11-13T18:23:44,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:23:44,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45483 2024-11-13T18:23:44,863 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45483 2024-11-13T18:23:44,882 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45483 2024-11-13T18:23:44,907 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45483 2024-11-13T18:23:44,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45483 2024-11-13T18:23:45,268 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/62ae331128c9:0 server-side Connection retries=45 2024-11-13T18:23:45,281 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:23:45,282 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:23:45,282 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:23:45,286 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:23:45,286 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:23:45,299 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:23:45,326 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:23:45,337 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39615 2024-11-13T18:23:45,341 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39615 connecting to ZooKeeper ensemble=127.0.0.1:63431 2024-11-13T18:23:45,344 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:23:45,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:23:45,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396150x0, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:23:45,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:396150x0, quorum=127.0.0.1:63431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:23:45,459 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39615-0x101369e15260001 connected 2024-11-13T18:23:45,476 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:23:45,535 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:23:45,546 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:23:45,570 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:23:45,590 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39615 2024-11-13T18:23:45,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39615 2024-11-13T18:23:45,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39615 2024-11-13T18:23:45,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39615 2024-11-13T18:23:45,655 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39615 2024-11-13T18:23:45,713 DEBUG [M:0;62ae331128c9:45483 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;62ae331128c9:45483 2024-11-13T18:23:45,727 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/62ae331128c9,45483,1731522221514 2024-11-13T18:23:45,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:23:45,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:23:45,767 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/62ae331128c9,45483,1731522221514 2024-11-13T18:23:45,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:23:45,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:45,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:45,863 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T18:23:45,883 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/62ae331128c9,45483,1731522221514 from backup master directory 2024-11-13T18:23:45,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:23:45,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/62ae331128c9,45483,1731522221514 2024-11-13T18:23:45,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:23:45,913 WARN [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:23:45,914 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=62ae331128c9,45483,1731522221514 2024-11-13T18:23:45,925 INFO [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T18:23:45,951 INFO [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T18:23:46,235 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/hbase.id] with ID: 5912eaa4-9875-4022-99ef-f25a03d8a257 2024-11-13T18:23:46,236 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/.tmp/hbase.id 2024-11-13T18:23:46,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741826_1002 (size=42) 2024-11-13T18:23:46,737 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/.tmp/hbase.id]:[hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/hbase.id] 2024-11-13T18:23:46,970 INFO [master/62ae331128c9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:23:46,997 INFO [master/62ae331128c9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T18:23:47,083 INFO [master/62ae331128c9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 80ms. 2024-11-13T18:23:47,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:47,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:47,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741827_1003 (size=196) 2024-11-13T18:23:47,312 INFO [master/62ae331128c9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:23:47,315 INFO [master/62ae331128c9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T18:23:47,392 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:47,412 INFO [master/62ae331128c9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:23:47,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741828_1004 (size=1189) 2024-11-13T18:23:47,669 INFO [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store 2024-11-13T18:23:47,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741829_1005 (size=34) 2024-11-13T18:23:48,260 INFO [master/62ae331128c9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T18:23:48,286 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:23:48,300 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:23:48,300 INFO [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:23:48,301 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:23:48,306 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:23:48,307 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:23:48,307 INFO [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:23:48,317 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731522228290Disabling compacts and flushes for region at 1731522228290Disabling writes for close at 1731522228306 (+16 ms)Writing region close event to WAL at 1731522228307 (+1 ms)Closed at 1731522228307 2024-11-13T18:23:48,331 WARN [master/62ae331128c9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/.initializing 2024-11-13T18:23:48,333 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/WALs/62ae331128c9,45483,1731522221514 2024-11-13T18:23:48,390 INFO [master/62ae331128c9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:23:48,435 INFO [master/62ae331128c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=62ae331128c9%2C45483%2C1731522221514, suffix=, logDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/WALs/62ae331128c9,45483,1731522221514, archiveDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/oldWALs, maxLogs=10 2024-11-13T18:23:48,483 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/WALs/62ae331128c9,45483,1731522221514/62ae331128c9%2C45483%2C1731522221514.1731522228445, exclude list is [], retry=0 2024-11-13T18:23:48,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46535,DS-cab5a945-efa3-4827-b04b-1ecd7edf5cb9,DISK] 2024-11-13T18:23:48,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T18:23:48,781 INFO [master/62ae331128c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/WALs/62ae331128c9,45483,1731522221514/62ae331128c9%2C45483%2C1731522221514.1731522228445 2024-11-13T18:23:48,783 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40951:40951)] 2024-11-13T18:23:48,789 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:23:48,790 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:23:48,805 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:48,807 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:48,943 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:49,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T18:23:49,086 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:49,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:23:49,094 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:49,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T18:23:49,135 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:49,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:23:49,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:49,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T18:23:49,161 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:49,168 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:23:49,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:49,190 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T18:23:49,191 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:49,200 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:23:49,205 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:49,214 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:49,223 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:49,267 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:49,268 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:49,284 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:23:49,305 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:23:49,351 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:23:49,369 INFO [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59390106, jitterRate=-0.11501845717430115}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:23:49,386 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731522228848Initializing all the Stores at 1731522228861 (+13 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522228862 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522228871 (+9 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522228871Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522228871Cleaning up temporary data from old regions at 1731522229269 (+398 ms)Region opened successfully at 1731522229385 (+116 ms) 2024-11-13T18:23:49,412 INFO [master/62ae331128c9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T18:23:49,591 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6884da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=62ae331128c9/172.17.0.3:0 2024-11-13T18:23:49,742 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T18:23:49,778 INFO [master/62ae331128c9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T18:23:49,778 INFO [master/62ae331128c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T18:23:49,788 INFO [master/62ae331128c9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:23:49,795 INFO [master/62ae331128c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 6 msec 2024-11-13T18:23:49,806 INFO [master/62ae331128c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-13T18:23:49,806 INFO [master/62ae331128c9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T18:23:49,915 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T18:23:49,949 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T18:23:50,003 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T18:23:50,012 INFO [master/62ae331128c9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T18:23:50,016 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T18:23:50,036 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T18:23:50,040 INFO [master/62ae331128c9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T18:23:50,098 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T18:23:50,124 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T18:23:50,134 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T18:23:50,152 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T18:23:50,202 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T18:23:50,211 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T18:23:50,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:23:50,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:23:50,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:50,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:50,243 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=62ae331128c9,45483,1731522221514, sessionid=0x101369e15260000, setting cluster-up flag (Was=false) 2024-11-13T18:23:50,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:50,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:50,420 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T18:23:50,442 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=62ae331128c9,45483,1731522221514 2024-11-13T18:23:50,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:50,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:50,639 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T18:23:50,647 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=62ae331128c9,45483,1731522221514 2024-11-13T18:23:50,675 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T18:23:50,773 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(746): ClusterId : 5912eaa4-9875-4022-99ef-f25a03d8a257 2024-11-13T18:23:50,779 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:23:50,801 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:23:50,801 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:23:50,825 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:23:50,826 DEBUG [RS:0;62ae331128c9:39615 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ee3c4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=62ae331128c9/172.17.0.3:0 2024-11-13T18:23:50,915 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;62ae331128c9:39615 2024-11-13T18:23:50,934 INFO [RS:0;62ae331128c9:39615 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:23:50,935 INFO [RS:0;62ae331128c9:39615 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:23:50,935 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:23:50,969 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(2659): reportForDuty to master=62ae331128c9,45483,1731522221514 with port=39615, startcode=1731522225114 2024-11-13T18:23:50,990 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:23:51,031 DEBUG [RS:0;62ae331128c9:39615 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:23:51,037 INFO [master/62ae331128c9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T18:23:51,097 INFO [master/62ae331128c9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T18:23:51,143 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 62ae331128c9,45483,1731522221514 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T18:23:51,208 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/62ae331128c9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:23:51,210 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/62ae331128c9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:23:51,212 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/62ae331128c9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:23:51,214 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/62ae331128c9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:23:51,219 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/62ae331128c9:0, corePoolSize=10, maxPoolSize=10 2024-11-13T18:23:51,219 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,220 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/62ae331128c9:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:23:51,220 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,260 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50157, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:23:51,269 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45483 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:51,315 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:23:51,316 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T18:23:51,340 INFO [master/62ae331128c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731522261340 2024-11-13T18:23:51,343 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T18:23:51,344 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T18:23:51,350 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T18:23:51,351 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T18:23:51,352 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T18:23:51,352 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T18:23:51,352 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:51,352 INFO [PEWorker-2 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T18:23:51,368 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T18:23:51,368 WARN [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T18:23:51,367 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,393 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T18:23:51,395 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T18:23:51,395 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T18:23:51,399 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T18:23:51,399 INFO [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T18:23:51,411 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/62ae331128c9:0:becomeActiveMaster-HFileCleaner.large.0-1731522231401,5,FailOnTimeoutGroup] 2024-11-13T18:23:51,429 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/62ae331128c9:0:becomeActiveMaster-HFileCleaner.small.0-1731522231412,5,FailOnTimeoutGroup] 2024-11-13T18:23:51,430 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,430 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T18:23:51,432 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,433 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,473 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(2659): reportForDuty to master=62ae331128c9,45483,1731522221514 with port=39615, startcode=1731522225114 2024-11-13T18:23:51,476 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45483 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 62ae331128c9,39615,1731522225114 2024-11-13T18:23:51,480 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45483 {}] master.ServerManager(517): Registering regionserver=62ae331128c9,39615,1731522225114 2024-11-13T18:23:51,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741831_1007 (size=1321) 2024-11-13T18:23:51,508 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b 2024-11-13T18:23:51,508 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33321 2024-11-13T18:23:51,508 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:23:51,515 INFO [PEWorker-2 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T18:23:51,516 INFO [PEWorker-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b 2024-11-13T18:23:51,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:23:51,549 DEBUG [RS:0;62ae331128c9:39615 {}] zookeeper.ZKUtil(111): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/62ae331128c9,39615,1731522225114 2024-11-13T18:23:51,549 WARN [RS:0;62ae331128c9:39615 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:23:51,549 INFO [RS:0;62ae331128c9:39615 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:23:51,549 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/WALs/62ae331128c9,39615,1731522225114 2024-11-13T18:23:51,559 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [62ae331128c9,39615,1731522225114] 2024-11-13T18:23:51,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741832_1008 (size=32) 2024-11-13T18:23:51,656 DEBUG [PEWorker-2 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:23:51,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:23:51,705 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:23:51,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:51,716 INFO [RS:0;62ae331128c9:39615 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:23:51,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:23:51,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:23:51,736 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:23:51,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:51,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:23:51,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:23:51,746 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:23:51,746 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:51,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:23:51,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:23:51,753 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:23:51,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:51,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:23:51,760 DEBUG [PEWorker-2 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:23:51,762 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740 2024-11-13T18:23:51,764 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740 2024-11-13T18:23:51,769 DEBUG [PEWorker-2 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:23:51,770 DEBUG [PEWorker-2 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:23:51,771 DEBUG [PEWorker-2 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:23:51,777 DEBUG [PEWorker-2 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:23:51,799 DEBUG [PEWorker-2 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:23:51,801 INFO [PEWorker-2 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69163161, jitterRate=0.03061141073703766}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:23:51,806 DEBUG [PEWorker-2 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731522231656Initializing all the Stores at 1731522231675 (+19 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522231675Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522231680 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522231680Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522231680Cleaning up temporary data from old regions at 1731522231770 (+90 ms)Region opened successfully at 1731522231806 (+36 ms) 2024-11-13T18:23:51,807 DEBUG [PEWorker-2 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:23:51,807 INFO [PEWorker-2 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:23:51,807 DEBUG [PEWorker-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:23:51,808 DEBUG [PEWorker-2 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:23:51,808 DEBUG [PEWorker-2 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:23:51,815 INFO [PEWorker-2 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:23:51,815 DEBUG [PEWorker-2 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731522231807Disabling compacts and flushes for region at 1731522231807Disabling writes for close at 1731522231808 (+1 ms)Writing region close event to WAL at 1731522231814 (+6 ms)Closed at 1731522231815 (+1 ms) 2024-11-13T18:23:51,817 INFO [RS:0;62ae331128c9:39615 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:23:51,824 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:23:51,824 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T18:23:51,830 INFO [RS:0;62ae331128c9:39615 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:23:51,830 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,836 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T18:23:51,841 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:23:51,853 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:23:51,877 INFO [RS:0;62ae331128c9:39615 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:23:51,882 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,884 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T18:23:51,893 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,893 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,894 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,894 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,904 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,905 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/62ae331128c9:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:23:51,905 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,905 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,905 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,905 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,906 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,906 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/62ae331128c9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:23:51,906 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/62ae331128c9:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:23:51,906 DEBUG [RS:0;62ae331128c9:39615 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:23:51,927 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,928 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,930 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,931 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,931 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,931 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=62ae331128c9,39615,1731522225114-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:23:51,988 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:23:51,992 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=62ae331128c9,39615,1731522225114-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,994 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:51,997 INFO [RS:0;62ae331128c9:39615 {}] regionserver.Replication(171): 62ae331128c9,39615,1731522225114 started 2024-11-13T18:23:52,042 WARN [62ae331128c9:45483 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T18:23:52,077 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:52,079 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1482): Serving as 62ae331128c9,39615,1731522225114, RpcServer on 62ae331128c9/172.17.0.3:39615, sessionid=0x101369e15260001 2024-11-13T18:23:52,088 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:23:52,089 DEBUG [RS:0;62ae331128c9:39615 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 62ae331128c9,39615,1731522225114 2024-11-13T18:23:52,090 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '62ae331128c9,39615,1731522225114' 2024-11-13T18:23:52,091 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:23:52,096 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:23:52,111 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:23:52,111 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:23:52,111 DEBUG [RS:0;62ae331128c9:39615 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 62ae331128c9,39615,1731522225114 2024-11-13T18:23:52,112 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '62ae331128c9,39615,1731522225114' 2024-11-13T18:23:52,112 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:23:52,115 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:23:52,116 DEBUG [RS:0;62ae331128c9:39615 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:23:52,116 INFO [RS:0;62ae331128c9:39615 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:23:52,117 INFO [RS:0;62ae331128c9:39615 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:23:52,225 INFO [RS:0;62ae331128c9:39615 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:23:52,243 INFO [RS:0;62ae331128c9:39615 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=62ae331128c9%2C39615%2C1731522225114, suffix=, logDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/WALs/62ae331128c9,39615,1731522225114, archiveDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/oldWALs, maxLogs=32 2024-11-13T18:23:52,302 DEBUG [RS:0;62ae331128c9:39615 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/WALs/62ae331128c9,39615,1731522225114/62ae331128c9%2C39615%2C1731522225114.1731522232248, exclude list is [], retry=0 2024-11-13T18:23:52,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46535,DS-cab5a945-efa3-4827-b04b-1ecd7edf5cb9,DISK] 2024-11-13T18:23:52,422 INFO [RS:0;62ae331128c9:39615 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/WALs/62ae331128c9,39615,1731522225114/62ae331128c9%2C39615%2C1731522225114.1731522232248 2024-11-13T18:23:52,432 DEBUG [RS:0;62ae331128c9:39615 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40951:40951)] 2024-11-13T18:23:52,546 DEBUG [62ae331128c9:45483 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T18:23:52,570 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:23:52,581 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 62ae331128c9,39615,1731522225114, state=OPENING 2024-11-13T18:23:52,648 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T18:23:52,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:52,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:23:52,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:23:52,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:23:52,673 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:23:52,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:23:52,952 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:23:52,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44721, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:23:53,052 INFO [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T18:23:53,052 INFO [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:23:53,053 INFO [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T18:23:53,077 INFO [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=62ae331128c9%2C39615%2C1731522225114.meta, suffix=.meta, logDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/WALs/62ae331128c9,39615,1731522225114, archiveDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/oldWALs, maxLogs=32 2024-11-13T18:23:53,144 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/WALs/62ae331128c9,39615,1731522225114/62ae331128c9%2C39615%2C1731522225114.meta.1731522233080.meta, exclude list is [], retry=0 2024-11-13T18:23:53,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46535,DS-cab5a945-efa3-4827-b04b-1ecd7edf5cb9,DISK] 2024-11-13T18:23:53,223 INFO [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/WALs/62ae331128c9,39615,1731522225114/62ae331128c9%2C39615%2C1731522225114.meta.1731522233080.meta 2024-11-13T18:23:53,239 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40951:40951)] 2024-11-13T18:23:53,240 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:23:53,243 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T18:23:53,248 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T18:23:53,260 INFO [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T18:23:53,283 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T18:23:53,286 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:23:53,287 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T18:23:53,287 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T18:23:53,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:23:53,318 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:23:53,318 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:53,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:23:53,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:23:53,330 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:23:53,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:53,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:23:53,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:23:53,336 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:23:53,336 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:53,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:23:53,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:23:53,343 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:23:53,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:53,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:23:53,346 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:23:53,354 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740 2024-11-13T18:23:53,364 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740 2024-11-13T18:23:53,374 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:23:53,374 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:23:53,380 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:23:53,392 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:23:53,401 INFO [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75222721, jitterRate=0.12090589106082916}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:23:53,402 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T18:23:53,408 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731522233288Writing region info on filesystem at 1731522233288Initializing all the Stores at 1731522233300 (+12 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522233300Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522233309 (+9 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522233309Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522233309Cleaning up temporary data from old regions at 1731522233375 (+66 ms)Running coprocessor post-open hooks at 1731522233402 (+27 ms)Region opened successfully at 1731522233408 (+6 ms) 2024-11-13T18:23:53,435 INFO [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731522232917 2024-11-13T18:23:53,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T18:23:53,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T18:23:53,498 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:23:53,498 DEBUG [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T18:23:53,499 INFO [RS_OPEN_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T18:23:53,506 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 62ae331128c9,39615,1731522225114, state=OPEN 2024-11-13T18:23:53,529 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:23:53,530 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T18:23:53,530 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T18:23:53,530 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T18:23:53,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:23:53,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:23:53,624 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:53,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:23:53,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:23:53,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T18:23:53,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=62ae331128c9,39615,1731522225114 in 940 msec 2024-11-13T18:23:53,660 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:23:53,660 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T18:23:53,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T18:23:53,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.8100 sec 2024-11-13T18:23:53,746 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:53,751 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:53,833 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:53,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39923, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:53,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 3.1350 sec 2024-11-13T18:23:53,907 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731522233906, completionTime=-1 2024-11-13T18:23:53,911 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T18:23:53,911 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:23:54,111 INFO [master/62ae331128c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-13T18:23:54,111 INFO [master/62ae331128c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731522294111 2024-11-13T18:23:54,111 INFO [master/62ae331128c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731522354111 2024-11-13T18:23:54,111 INFO [master/62ae331128c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 200 msec 2024-11-13T18:23:54,117 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=62ae331128c9,45483,1731522221514-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:54,118 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=62ae331128c9,45483,1731522221514-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:54,135 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=62ae331128c9,45483,1731522221514-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:54,138 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-62ae331128c9:45483, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:54,168 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:54,183 DEBUG [master/62ae331128c9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T18:23:54,184 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:54,275 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 8.361sec 2024-11-13T18:23:54,277 INFO [master/62ae331128c9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T18:23:54,284 INFO [master/62ae331128c9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T18:23:54,285 INFO [master/62ae331128c9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T18:23:54,286 INFO [master/62ae331128c9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T18:23:54,286 INFO [master/62ae331128c9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T18:23:54,295 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=62ae331128c9,45483,1731522221514-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:23:54,296 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=62ae331128c9,45483,1731522221514-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T18:23:54,350 DEBUG [master/62ae331128c9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T18:23:54,351 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T18:23:54,352 INFO [master/62ae331128c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=62ae331128c9,45483,1731522221514-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:23:54,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e56da13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:54,377 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T18:23:54,377 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T18:23:54,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:54,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:54,427 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3162b06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:54,461 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:54,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@118af296, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:54,481 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:54,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:54,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:54,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=62ae331128c9,45483,1731522221514 2024-11-13T18:23:54,617 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=209, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=2002, ProcessCount=11, AvailableMemoryMB=1893 2024-11-13T18:23:54,683 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:23:54,706 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(321): The fetched master address is 62ae331128c9,45483,1731522221514 2024-11-13T18:23:54,709 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ce862ed 2024-11-13T18:23:54,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:23:54,721 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:23:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:23:54,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:23:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:23:54,799 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:23:54,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-13T18:23:54,805 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:23:54,820 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:23:54,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741835_1011 (size=1061) 2024-11-13T18:23:54,946 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b 2024-11-13T18:23:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:23:55,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741836_1012 (size=53) 2024-11-13T18:23:55,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:23:55,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing debb09c76feaf68052a52c8303bbb905, disabling compactions & flushes 2024-11-13T18:23:55,050 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:55,050 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:55,050 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. after waiting 0 ms 2024-11-13T18:23:55,050 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:55,050 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:55,050 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for debb09c76feaf68052a52c8303bbb905: Waiting for close lock at 1731522235049Disabling compacts and flushes for region at 1731522235049Disabling writes for close at 1731522235050 (+1 ms)Writing region close event to WAL at 1731522235050Closed at 1731522235050 2024-11-13T18:23:55,065 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:23:55,093 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731522235066"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731522235066"}]},"ts":"1731522235066"} 2024-11-13T18:23:55,111 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:23:55,116 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:23:55,129 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522235116"}]},"ts":"1731522235116"} 2024-11-13T18:23:55,145 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:23:55,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=debb09c76feaf68052a52c8303bbb905, ASSIGN}] 2024-11-13T18:23:55,152 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=debb09c76feaf68052a52c8303bbb905, ASSIGN 2024-11-13T18:23:55,163 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=debb09c76feaf68052a52c8303bbb905, ASSIGN; state=OFFLINE, location=62ae331128c9,39615,1731522225114; forceNewPlan=false, retain=false 2024-11-13T18:23:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:23:55,324 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=debb09c76feaf68052a52c8303bbb905, regionState=OPENING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:23:55,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=debb09c76feaf68052a52c8303bbb905, ASSIGN because future has completed 2024-11-13T18:23:55,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:23:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:23:55,574 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:55,575 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:23:55,588 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,588 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:23:55,588 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,589 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,605 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,630 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:23:55,631 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region debb09c76feaf68052a52c8303bbb905 columnFamilyName A 2024-11-13T18:23:55,631 DEBUG [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:55,647 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] regionserver.HStore(327): Store=debb09c76feaf68052a52c8303bbb905/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:23:55,648 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,663 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:23:55,664 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region debb09c76feaf68052a52c8303bbb905 columnFamilyName B 2024-11-13T18:23:55,664 DEBUG [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:55,666 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] regionserver.HStore(327): Store=debb09c76feaf68052a52c8303bbb905/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:23:55,666 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,671 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:23:55,672 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region debb09c76feaf68052a52c8303bbb905 columnFamilyName C 2024-11-13T18:23:55,672 DEBUG [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:23:55,679 INFO [StoreOpener-debb09c76feaf68052a52c8303bbb905-1 {}] regionserver.HStore(327): Store=debb09c76feaf68052a52c8303bbb905/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:23:55,680 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:55,681 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,697 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,707 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,735 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,735 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,740 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:23:55,773 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,787 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:23:55,789 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened debb09c76feaf68052a52c8303bbb905; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59269263, jitterRate=-0.11681915819644928}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:23:55,789 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:55,791 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for debb09c76feaf68052a52c8303bbb905: Running coprocessor pre-open hook at 1731522235589Writing region info on filesystem at 1731522235589Initializing all the Stores at 1731522235595 (+6 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522235595Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522235599 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522235599Cleaning up temporary data from old regions at 1731522235735 (+136 ms)Running coprocessor post-open hooks at 1731522235789 (+54 ms)Region opened successfully at 1731522235791 (+2 ms) 2024-11-13T18:23:55,795 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., pid=6, masterSystemTime=1731522235521 2024-11-13T18:23:55,801 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:55,801 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:55,804 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=debb09c76feaf68052a52c8303bbb905, regionState=OPEN, openSeqNum=2, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:23:55,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:23:55,822 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45483 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=62ae331128c9,39615,1731522225114, table=TestAcidGuarantees, region=debb09c76feaf68052a52c8303bbb905. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:23:55,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T18:23:55,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T18:23:55,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=debb09c76feaf68052a52c8303bbb905, ASSIGN in 788 msec 2024-11-13T18:23:55,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 in 544 msec 2024-11-13T18:23:55,975 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:23:55,976 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522235975"}]},"ts":"1731522235975"} 2024-11-13T18:23:55,993 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:23:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:23:55,999 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:23:56,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2430 sec 2024-11-13T18:23:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:23:57,040 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:23:57,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@649aef2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:57,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:57,069 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:57,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:57,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10817e68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:57,083 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70b741f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,087 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:57,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:57,132 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:23:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 62ae331128c9,45483,1731522221514 2024-11-13T18:23:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@645b8251 2024-11-13T18:23:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:23:57,154 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:23:57,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d30157, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:57,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:57,184 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@80a4ffc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:57,211 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d58cc8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a72b63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,229 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:57,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:57,256 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:57,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555485ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:57,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:57,273 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b6c7e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c8c49bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:57,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:57,304 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d14bbbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:57,323 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf9ec4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309f8170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:57,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:57,350 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f29268d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:57,387 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:57,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45bdfbac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac76ce1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:57,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:57,403 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@726ebe64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:57,415 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51749f3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5864d7ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:57,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:57,435 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13bb7624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:57,445 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e993e6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2299c977, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:57,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:57,466 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:57,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:57,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c8fee02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:57,476 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587231a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1288062b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:57,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:57,493 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24f290d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:57,503 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5008e93b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672a225b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:23:57,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:23:57,544 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:23:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:23:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:23:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4800a3d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:23:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:23:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:23:57,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:23:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@584ff6eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:23:57,574 DEBUG [Thread-146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:57,583 DEBUG [Thread-146 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:57,584 DEBUG [Thread-146 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:57,587 DEBUG [Thread-159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:57,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:57,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:57,599 DEBUG [Thread-148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:23:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:23:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:57,608 DEBUG [Thread-152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:57,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:57,615 DEBUG [Thread-157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:57,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:23:57,631 DEBUG [Thread-152 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:57,632 DEBUG [Thread-152 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:57,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:57,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:57,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:23:57,637 DEBUG [Thread-150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:57,643 DEBUG [Thread-161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:57,647 DEBUG [Thread-144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:57,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:57,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:57,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:23:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:57,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:57,662 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T18:23:57,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:57,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:57,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:57,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:23:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:23:57,685 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T18:23:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:23:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:23:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-13T18:23:57,717 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:23:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:23:57,721 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:23:57,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:23:57,724 DEBUG [Thread-155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:23:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:23:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:23:57,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:23:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:23:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:23:57,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:23:57,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:23:57,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:23:57,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:23:57,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:23:57,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:23:57,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:23:57,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:23:57,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:23:57,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:57,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:57,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:23:58,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:23:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:23:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:58,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522298250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:23:58,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:23:58,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522238198, max timestamp=1731522238283Num uniques 859; 2024-11-13T18:23:58,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:58,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:23:58,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:58,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522238198, max timestamp=1731522238283Num uniques 859; 2024-11-13T18:23:58,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:23:58,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:23:58,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522238198, max timestamp=1731522238283Num uniques 859; 2024-11-13T18:23:58,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:23:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/08b0ec93bbbe4b689bfb271cf98ba611 is 50, key is test_row_0/A:col10/1731522237791/Put/seqid=0 2024-11-13T18:23:58,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:23:58,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:58,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:23:58,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:58,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,709 INFO [AsyncFSWAL-0-hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData-prefix:62ae331128c9,45483,1731522221514 {}] wal.AbstractFSWAL(1368): Slow sync cost: 106 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46535,DS-cab5a945-efa3-4827-b04b-1ecd7edf5cb9,DISK]] 2024-11-13T18:23:58,718 INFO [AsyncFSWAL-0-hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b-prefix:62ae331128c9,39615,1731522225114 {}] wal.AbstractFSWAL(1368): Slow sync cost: 122 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46535,DS-cab5a945-efa3-4827-b04b-1ecd7edf5cb9,DISK]] 2024-11-13T18:23:58,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741837_1013 (size=12001) 2024-11-13T18:23:58,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/08b0ec93bbbe4b689bfb271cf98ba611 2024-11-13T18:23:58,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522298763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522298778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:23:58,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:23:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:23:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:58,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:58,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522298793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:58,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522298813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:58,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:58,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522298821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:58,954 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:23:59,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522298942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522298964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:23:59,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:59,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:23:59,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:59,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/320813ab385b40f68bbb90d345294d59 is 50, key is test_row_0/B:col10/1731522237791/Put/seqid=0 2024-11-13T18:23:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522299051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741838_1014 (size=12001) 2024-11-13T18:23:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522299055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522299055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/320813ab385b40f68bbb90d345294d59 2024-11-13T18:23:59,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:23:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:23:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:59,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522299224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/caed061ad3cc4ff3b45572e114c3821e is 50, key is test_row_0/C:col10/1731522237791/Put/seqid=0 2024-11-13T18:23:59,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522299244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522299361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522299363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:23:59,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:59,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:23:59,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:59,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522299367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741839_1015 (size=12001) 2024-11-13T18:23:59,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/caed061ad3cc4ff3b45572e114c3821e 2024-11-13T18:23:59,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/08b0ec93bbbe4b689bfb271cf98ba611 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/08b0ec93bbbe4b689bfb271cf98ba611 2024-11-13T18:23:59,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/08b0ec93bbbe4b689bfb271cf98ba611, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:23:59,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/320813ab385b40f68bbb90d345294d59 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/320813ab385b40f68bbb90d345294d59 2024-11-13T18:23:59,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:23:59,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:59,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:23:59,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:59,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:23:59,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/320813ab385b40f68bbb90d345294d59, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:23:59,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/caed061ad3cc4ff3b45572e114c3821e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/caed061ad3cc4ff3b45572e114c3821e 2024-11-13T18:23:59,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/caed061ad3cc4ff3b45572e114c3821e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:23:59,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for debb09c76feaf68052a52c8303bbb905 in 1866ms, sequenceid=13, compaction requested=false 2024-11-13T18:23:59,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:23:59,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:23:59,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:23:59,729 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-13T18:23:59,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:23:59,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:23:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:23:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:23:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:23:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:23:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:23:59,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:23:59,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/50b3c304a68a47f690fa6c333f9127f6 is 50, key is test_row_0/A:col10/1731522238750/Put/seqid=0 2024-11-13T18:23:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:23:59,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522299806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522299843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:23:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522299854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:23:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:23:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:23:59,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741840_1016 (size=14341) 2024-11-13T18:24:00,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522299891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522299908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522300021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522300028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522300029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522300137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522300137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,348 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/50b3c304a68a47f690fa6c333f9127f6 2024-11-13T18:24:00,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522300328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522300328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522300339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/e05e9d4c91734ee4916ef8c299124d28 is 50, key is test_row_0/B:col10/1731522238750/Put/seqid=0 2024-11-13T18:24:00,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522300387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522300401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741841_1017 (size=12001) 2024-11-13T18:24:00,517 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/e05e9d4c91734ee4916ef8c299124d28 2024-11-13T18:24:00,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/7b0fd4c61c7741dd9bbb5c3904f22f81 is 50, key is test_row_0/C:col10/1731522238750/Put/seqid=0 2024-11-13T18:24:00,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741842_1018 (size=12001) 2024-11-13T18:24:00,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522300682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522300707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522300739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522300748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:00,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522300757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,068 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/7b0fd4c61c7741dd9bbb5c3904f22f81 2024-11-13T18:24:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/50b3c304a68a47f690fa6c333f9127f6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/50b3c304a68a47f690fa6c333f9127f6 2024-11-13T18:24:01,122 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/50b3c304a68a47f690fa6c333f9127f6, entries=200, sequenceid=49, filesize=14.0 K 2024-11-13T18:24:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/e05e9d4c91734ee4916ef8c299124d28 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e05e9d4c91734ee4916ef8c299124d28 2024-11-13T18:24:01,158 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e05e9d4c91734ee4916ef8c299124d28, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T18:24:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/7b0fd4c61c7741dd9bbb5c3904f22f81 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/7b0fd4c61c7741dd9bbb5c3904f22f81 2024-11-13T18:24:01,226 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/7b0fd4c61c7741dd9bbb5c3904f22f81, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T18:24:01,228 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for debb09c76feaf68052a52c8303bbb905 in 1500ms, sequenceid=49, compaction requested=false 2024-11-13T18:24:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T18:24:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T18:24:01,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T18:24:01,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.5780 sec 2024-11-13T18:24:01,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5100 sec 2024-11-13T18:24:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:01,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:24:01,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:01,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:01,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:01,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:01,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:01,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:01,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/c9b6225411854ae38e2bc95e83bf1f4b is 50, key is test_row_0/A:col10/1731522239888/Put/seqid=0 2024-11-13T18:24:01,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741843_1019 (size=16681) 2024-11-13T18:24:01,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522301612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522301658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522301669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:01,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522301680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522301682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:01,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522301799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/c9b6225411854ae38e2bc95e83bf1f4b 2024-11-13T18:24:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:24:01,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:01,893 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522301839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-13T18:24:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:24:01,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/cf06300a43d94e38867db8d641caede6 is 50, key is test_row_0/B:col10/1731522239888/Put/seqid=0 2024-11-13T18:24:01,915 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:01,919 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:01,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:01,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522301860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522301860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:01,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522301871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:01,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741844_1020 (size=12001) 2024-11-13T18:24:01,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/cf06300a43d94e38867db8d641caede6 2024-11-13T18:24:02,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/e43a46b79baa44d1bd4eca8eab89a6d9 is 50, key is test_row_0/C:col10/1731522239888/Put/seqid=0 2024-11-13T18:24:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:24:02,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522302017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741845_1021 (size=12001) 2024-11-13T18:24:02,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/e43a46b79baa44d1bd4eca8eab89a6d9 2024-11-13T18:24:02,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T18:24:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:02,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:02,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/c9b6225411854ae38e2bc95e83bf1f4b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/c9b6225411854ae38e2bc95e83bf1f4b 2024-11-13T18:24:02,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/c9b6225411854ae38e2bc95e83bf1f4b, entries=250, sequenceid=62, filesize=16.3 K 2024-11-13T18:24:02,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522302108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522302136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/cf06300a43d94e38867db8d641caede6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cf06300a43d94e38867db8d641caede6 2024-11-13T18:24:02,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cf06300a43d94e38867db8d641caede6, entries=150, sequenceid=62, filesize=11.7 K 2024-11-13T18:24:02,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522302147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522302147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/e43a46b79baa44d1bd4eca8eab89a6d9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/e43a46b79baa44d1bd4eca8eab89a6d9 2024-11-13T18:24:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:24:02,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/e43a46b79baa44d1bd4eca8eab89a6d9, entries=150, sequenceid=62, filesize=11.7 K 2024-11-13T18:24:02,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for debb09c76feaf68052a52c8303bbb905 in 938ms, sequenceid=62, compaction requested=true 2024-11-13T18:24:02,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:02,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T18:24:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:02,252 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:24:02,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:02,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:02,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:02,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:02,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:02,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:02,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:02,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:24:02,263 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:02,264 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:02,267 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:02,270 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/B is initiating minor compaction (all files) 2024-11-13T18:24:02,271 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/B in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:02,272 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/320813ab385b40f68bbb90d345294d59, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e05e9d4c91734ee4916ef8c299124d28, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cf06300a43d94e38867db8d641caede6] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=35.2 K 2024-11-13T18:24:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/2d041312db054064877ca15c9fc76b1a is 50, key is test_row_0/A:col10/1731522241641/Put/seqid=0 2024-11-13T18:24:02,274 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:02,275 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/A is initiating minor compaction (all files) 2024-11-13T18:24:02,275 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/A in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:02,275 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/08b0ec93bbbe4b689bfb271cf98ba611, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/50b3c304a68a47f690fa6c333f9127f6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/c9b6225411854ae38e2bc95e83bf1f4b] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=42.0 K 2024-11-13T18:24:02,276 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 320813ab385b40f68bbb90d345294d59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731522237791 2024-11-13T18:24:02,277 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08b0ec93bbbe4b689bfb271cf98ba611, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731522237791 2024-11-13T18:24:02,278 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e05e9d4c91734ee4916ef8c299124d28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731522238750 2024-11-13T18:24:02,279 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50b3c304a68a47f690fa6c333f9127f6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731522238750 2024-11-13T18:24:02,280 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting cf06300a43d94e38867db8d641caede6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522239888 2024-11-13T18:24:02,280 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9b6225411854ae38e2bc95e83bf1f4b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522239841 2024-11-13T18:24:02,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741846_1022 (size=12001) 2024-11-13T18:24:02,330 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/2d041312db054064877ca15c9fc76b1a 2024-11-13T18:24:02,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:02,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/cd4aac6c0170482fb0d0ee6003fc96ad is 50, key is test_row_0/B:col10/1731522241641/Put/seqid=0 2024-11-13T18:24:02,438 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#B#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:02,440 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/4c403657df9b4303bba3a50fe209633b is 50, key is test_row_0/B:col10/1731522239888/Put/seqid=0 2024-11-13T18:24:02,451 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#A#compaction#10 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:02,452 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/7b0e6b9da4f24fcd82dfeba35b56110e is 50, key is test_row_0/A:col10/1731522239888/Put/seqid=0 2024-11-13T18:24:02,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741847_1023 (size=12001) 2024-11-13T18:24:02,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741848_1024 (size=12104) 2024-11-13T18:24:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:24:02,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741849_1025 (size=12104) 2024-11-13T18:24:02,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522302531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522302572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522302583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522302587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522302596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522302727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522302731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522302737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522302743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:02,891 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/cd4aac6c0170482fb0d0ee6003fc96ad 2024-11-13T18:24:02,951 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/4c403657df9b4303bba3a50fe209633b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/4c403657df9b4303bba3a50fe209633b 2024-11-13T18:24:02,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/6d51249f5e8649a4a8513c64de37b854 is 50, key is test_row_0/C:col10/1731522241641/Put/seqid=0 2024-11-13T18:24:02,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522302993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741850_1026 (size=12001) 2024-11-13T18:24:03,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,015 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/7b0e6b9da4f24fcd82dfeba35b56110e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/7b0e6b9da4f24fcd82dfeba35b56110e 2024-11-13T18:24:03,017 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/6d51249f5e8649a4a8513c64de37b854 2024-11-13T18:24:03,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,020 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/B of debb09c76feaf68052a52c8303bbb905 into 4c403657df9b4303bba3a50fe209633b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522302995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,020 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522302995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522302995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,029 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/B, priority=13, startTime=1731522242260; duration=0sec 2024-11-13T18:24:03,037 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:03,038 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:B 2024-11-13T18:24:03,038 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:24:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/2d041312db054064877ca15c9fc76b1a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2d041312db054064877ca15c9fc76b1a 2024-11-13T18:24:03,061 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/A of debb09c76feaf68052a52c8303bbb905 into 7b0e6b9da4f24fcd82dfeba35b56110e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:03,061 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:03,061 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/A, priority=13, startTime=1731522242248; duration=0sec 2024-11-13T18:24:03,061 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:03,061 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:A 2024-11-13T18:24:03,064 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:03,065 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/C is initiating minor compaction (all files) 2024-11-13T18:24:03,065 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/C in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:03,065 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/caed061ad3cc4ff3b45572e114c3821e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/7b0fd4c61c7741dd9bbb5c3904f22f81, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/e43a46b79baa44d1bd4eca8eab89a6d9] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=35.2 K 2024-11-13T18:24:03,073 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting caed061ad3cc4ff3b45572e114c3821e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731522237791 2024-11-13T18:24:03,077 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0fd4c61c7741dd9bbb5c3904f22f81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731522238750 2024-11-13T18:24:03,080 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e43a46b79baa44d1bd4eca8eab89a6d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522239888 2024-11-13T18:24:03,088 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2d041312db054064877ca15c9fc76b1a, entries=150, sequenceid=82, filesize=11.7 K 2024-11-13T18:24:03,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/cd4aac6c0170482fb0d0ee6003fc96ad as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cd4aac6c0170482fb0d0ee6003fc96ad 2024-11-13T18:24:03,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522303107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,143 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#C#compaction#14 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:03,144 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/82132888f8c641e78fa1e839bde93101 is 50, key is test_row_0/C:col10/1731522239888/Put/seqid=0 2024-11-13T18:24:03,163 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cd4aac6c0170482fb0d0ee6003fc96ad, entries=150, sequenceid=82, filesize=11.7 K 2024-11-13T18:24:03,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/6d51249f5e8649a4a8513c64de37b854 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/6d51249f5e8649a4a8513c64de37b854 2024-11-13T18:24:03,199 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/6d51249f5e8649a4a8513c64de37b854, entries=150, sequenceid=82, filesize=11.7 K 2024-11-13T18:24:03,201 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for debb09c76feaf68052a52c8303bbb905 in 950ms, sequenceid=82, compaction requested=false 2024-11-13T18:24:03,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:03,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:03,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T18:24:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T18:24:03,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741851_1027 (size=12104) 2024-11-13T18:24:03,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T18:24:03,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2930 sec 2024-11-13T18:24:03,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3390 sec 2024-11-13T18:24:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:03,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T18:24:03,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:03,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/31c1904de95148368a2a483bcacabcff is 50, key is test_row_0/A:col10/1731522243311/Put/seqid=0 2024-11-13T18:24:03,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741852_1028 (size=14341) 2024-11-13T18:24:03,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/31c1904de95148368a2a483bcacabcff 2024-11-13T18:24:03,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/36aef2fd3e6d423fa7e7c7262fb55e05 is 50, key is test_row_0/B:col10/1731522243311/Put/seqid=0 2024-11-13T18:24:03,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741853_1029 (size=12001) 2024-11-13T18:24:03,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/36aef2fd3e6d423fa7e7c7262fb55e05 2024-11-13T18:24:03,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:24:03,479 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T18:24:03,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522303461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522303475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522303477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/70bd4188c9cf4657867a71a0c678daf6 is 50, key is test_row_0/C:col10/1731522243311/Put/seqid=0 2024-11-13T18:24:03,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522303477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741854_1030 (size=12001) 2024-11-13T18:24:03,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522303597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522303599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522303606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522303616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,695 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/82132888f8c641e78fa1e839bde93101 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/82132888f8c641e78fa1e839bde93101 2024-11-13T18:24:03,739 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/C of debb09c76feaf68052a52c8303bbb905 into 82132888f8c641e78fa1e839bde93101(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:03,739 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:03,739 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/C, priority=13, startTime=1731522242260; duration=0sec 2024-11-13T18:24:03,740 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:03,740 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:C 2024-11-13T18:24:03,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522303821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522303824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522303827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522303827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:03,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/70bd4188c9cf4657867a71a0c678daf6 2024-11-13T18:24:03,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/31c1904de95148368a2a483bcacabcff as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/31c1904de95148368a2a483bcacabcff 2024-11-13T18:24:04,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/31c1904de95148368a2a483bcacabcff, entries=200, sequenceid=98, filesize=14.0 K 2024-11-13T18:24:04,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/36aef2fd3e6d423fa7e7c7262fb55e05 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/36aef2fd3e6d423fa7e7c7262fb55e05 2024-11-13T18:24:04,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/36aef2fd3e6d423fa7e7c7262fb55e05, entries=150, sequenceid=98, filesize=11.7 K 2024-11-13T18:24:04,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/70bd4188c9cf4657867a71a0c678daf6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/70bd4188c9cf4657867a71a0c678daf6 2024-11-13T18:24:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:24:04,058 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:04,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-13T18:24:04,063 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:24:04,065 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:04,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:04,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/70bd4188c9cf4657867a71a0c678daf6, entries=150, sequenceid=98, filesize=11.7 K 2024-11-13T18:24:04,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for debb09c76feaf68052a52c8303bbb905 in 751ms, sequenceid=98, compaction requested=true 2024-11-13T18:24:04,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:04,076 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:04,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:04,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:04,077 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:04,081 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:04,081 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/A is initiating minor compaction (all files) 2024-11-13T18:24:04,081 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/A in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,082 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/7b0e6b9da4f24fcd82dfeba35b56110e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2d041312db054064877ca15c9fc76b1a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/31c1904de95148368a2a483bcacabcff] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=37.5 K 2024-11-13T18:24:04,082 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:04,083 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/B is initiating minor compaction (all files) 2024-11-13T18:24:04,083 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/B in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,083 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/4c403657df9b4303bba3a50fe209633b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cd4aac6c0170482fb0d0ee6003fc96ad, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/36aef2fd3e6d423fa7e7c7262fb55e05] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=35.3 K 2024-11-13T18:24:04,084 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0e6b9da4f24fcd82dfeba35b56110e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522239888 2024-11-13T18:24:04,086 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c403657df9b4303bba3a50fe209633b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522239888 2024-11-13T18:24:04,086 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d041312db054064877ca15c9fc76b1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731522241593 2024-11-13T18:24:04,087 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting cd4aac6c0170482fb0d0ee6003fc96ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731522241593 2024-11-13T18:24:04,088 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31c1904de95148368a2a483bcacabcff, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522242492 2024-11-13T18:24:04,089 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 36aef2fd3e6d423fa7e7c7262fb55e05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522242492 2024-11-13T18:24:04,134 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:04,135 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/07851e8e62884969a2f7ce22f34af570 is 50, key is test_row_0/B:col10/1731522243311/Put/seqid=0 2024-11-13T18:24:04,150 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:04,155 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/698cbd2877a14ffc9e59b97b97bfcd5f is 50, key is test_row_0/A:col10/1731522243311/Put/seqid=0 2024-11-13T18:24:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:04,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:24:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:24:04,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741855_1031 (size=12207) 2024-11-13T18:24:04,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:24:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522243441, max timestamp=1731522244163Num uniques 859; 2024-11-13T18:24:04,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:24:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,238 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/07851e8e62884969a2f7ce22f34af570 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/07851e8e62884969a2f7ce22f34af570 2024-11-13T18:24:04,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522243441, max timestamp=1731522244163Num uniques 859; 2024-11-13T18:24:04,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:24:04,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522243441, max timestamp=1731522244163Num uniques 859; 2024-11-13T18:24:04,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:24:04,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:04,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:04,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:04,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:04,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:04,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:04,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741856_1032 (size=12207) 2024-11-13T18:24:04,281 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/B of debb09c76feaf68052a52c8303bbb905 into 07851e8e62884969a2f7ce22f34af570(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:04,281 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:04,281 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/B, priority=13, startTime=1731522244077; duration=0sec 2024-11-13T18:24:04,282 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:04,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/6272981d9d62439f911a9a9fc0bb2588 is 50, key is test_row_0/A:col10/1731522244161/Put/seqid=0 2024-11-13T18:24:04,282 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:B 2024-11-13T18:24:04,282 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:04,294 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:04,294 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/C is initiating minor compaction (all files) 2024-11-13T18:24:04,295 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/C in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,295 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/82132888f8c641e78fa1e839bde93101, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/6d51249f5e8649a4a8513c64de37b854, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/70bd4188c9cf4657867a71a0c678daf6] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=35.3 K 2024-11-13T18:24:04,307 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 82132888f8c641e78fa1e839bde93101, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522239888 2024-11-13T18:24:04,308 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d51249f5e8649a4a8513c64de37b854, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731522241593 2024-11-13T18:24:04,310 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 70bd4188c9cf4657867a71a0c678daf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522242492 2024-11-13T18:24:04,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741857_1033 (size=12001) 2024-11-13T18:24:04,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/6272981d9d62439f911a9a9fc0bb2588 2024-11-13T18:24:04,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/a4e6556441de4d5a96f9616e07d1e3fa is 50, key is test_row_0/B:col10/1731522244161/Put/seqid=0 2024-11-13T18:24:04,375 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:04,377 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/257400b10abd4d8daeac0e9efc3caf73 is 50, key is test_row_0/C:col10/1731522243311/Put/seqid=0 2024-11-13T18:24:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:24:04,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:24:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522244320, max timestamp=1731522244375Num uniques 859; 2024-11-13T18:24:04,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:04,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741858_1034 (size=12001) 2024-11-13T18:24:04,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/a4e6556441de4d5a96f9616e07d1e3fa 2024-11-13T18:24:04,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741859_1035 (size=12207) 2024-11-13T18:24:04,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522244320, max timestamp=1731522244375Num uniques 859; 2024-11-13T18:24:04,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:04,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522244320, max timestamp=1731522244418Num uniques 859; 2024-11-13T18:24:04,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:24:04,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/64d950561bd74a7384dc78a4b3895c97 is 50, key is test_row_0/C:col10/1731522244161/Put/seqid=0 2024-11-13T18:24:04,531 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/257400b10abd4d8daeac0e9efc3caf73 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/257400b10abd4d8daeac0e9efc3caf73 2024-11-13T18:24:04,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741860_1036 (size=12001) 2024-11-13T18:24:04,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/64d950561bd74a7384dc78a4b3895c97 2024-11-13T18:24:04,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:24:04,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:04,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/6272981d9d62439f911a9a9fc0bb2588 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/6272981d9d62439f911a9a9fc0bb2588 2024-11-13T18:24:04,588 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/C of debb09c76feaf68052a52c8303bbb905 into 257400b10abd4d8daeac0e9efc3caf73(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:04,588 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:04,589 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/C, priority=13, startTime=1731522244078; duration=0sec 2024-11-13T18:24:04,589 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:04,589 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:C 2024-11-13T18:24:04,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/6272981d9d62439f911a9a9fc0bb2588, entries=150, sequenceid=120, filesize=11.7 K 2024-11-13T18:24:04,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/a4e6556441de4d5a96f9616e07d1e3fa as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/a4e6556441de4d5a96f9616e07d1e3fa 2024-11-13T18:24:04,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/a4e6556441de4d5a96f9616e07d1e3fa, entries=150, sequenceid=120, filesize=11.7 K 2024-11-13T18:24:04,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731522244507, max timestamp=1731522244631Num uniques 1018; 2024-11-13T18:24:04,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:24:04,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/64d950561bd74a7384dc78a4b3895c97 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/64d950561bd74a7384dc78a4b3895c97 2024-11-13T18:24:04,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731522244507, max timestamp=1731522244631Num uniques 1009; 2024-11-13T18:24:04,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-13T18:24:04,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522244507, max timestamp=1731522244615Num uniques 1018; 2024-11-13T18:24:04,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-13T18:24:04,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/64d950561bd74a7384dc78a4b3895c97, entries=150, sequenceid=120, filesize=11.7 K 2024-11-13T18:24:04,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=52.87 KB/54142 for debb09c76feaf68052a52c8303bbb905 in 511ms, sequenceid=120, compaction requested=false 2024-11-13T18:24:04,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:04,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=59.58 KB heapSize=163.17 KB 2024-11-13T18:24:04,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:04,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:04,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:04,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:04,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:04,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:24:04,705 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/698cbd2877a14ffc9e59b97b97bfcd5f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/698cbd2877a14ffc9e59b97b97bfcd5f 2024-11-13T18:24:04,725 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/A of debb09c76feaf68052a52c8303bbb905 into 698cbd2877a14ffc9e59b97b97bfcd5f(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:04,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:24:04,725 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:04,725 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/A, priority=13, startTime=1731522244076; duration=0sec 2024-11-13T18:24:04,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:04,725 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:04,725 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:A 2024-11-13T18:24:04,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/2af99cd069fb472493c16ef1f359f387 is 50, key is test_row_0/A:col10/1731522244641/Put/seqid=0 2024-11-13T18:24:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741861_1037 (size=14541) 2024-11-13T18:24:04,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.33 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/2af99cd069fb472493c16ef1f359f387 2024-11-13T18:24:04,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:04,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522304812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:04,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:04,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522304814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:04,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:04,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:04,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:04,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522244795, max timestamp=1731522244816Num uniques 859; 2024-11-13T18:24:04,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:24:04,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522244795, max timestamp=1731522244816Num uniques 859; 2024-11-13T18:24:04,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522244795, max timestamp=1731522244816Num uniques 859; 2024-11-13T18:24:04,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:24:04,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:24:04,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/efc16c7f436848f68e0e2044a5ca770b is 50, key is test_row_0/B:col10/1731522244641/Put/seqid=0 2024-11-13T18:24:04,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:24:04,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:04,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:04,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741862_1038 (size=12151) 2024-11-13T18:24:04,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/efc16c7f436848f68e0e2044a5ca770b 2024-11-13T18:24:04,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/fe0a0ecd893e49848391d46d9ec485bf is 50, key is test_row_0/C:col10/1731522244641/Put/seqid=0 2024-11-13T18:24:04,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522304961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:04,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522304966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:04,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522304966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522304992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522304992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741863_1039 (size=12151) 2024-11-13T18:24:05,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/fe0a0ecd893e49848391d46d9ec485bf 2024-11-13T18:24:05,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:24:05,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:05,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:05,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:05,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:05,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:05,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/2af99cd069fb472493c16ef1f359f387 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2af99cd069fb472493c16ef1f359f387 2024-11-13T18:24:05,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522305104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2af99cd069fb472493c16ef1f359f387, entries=200, sequenceid=165, filesize=14.2 K 2024-11-13T18:24:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522305107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522305107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/efc16c7f436848f68e0e2044a5ca770b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/efc16c7f436848f68e0e2044a5ca770b 2024-11-13T18:24:05,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/efc16c7f436848f68e0e2044a5ca770b, entries=150, sequenceid=165, filesize=11.9 K 2024-11-13T18:24:05,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522305117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522305119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/fe0a0ecd893e49848391d46d9ec485bf as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fe0a0ecd893e49848391d46d9ec485bf 2024-11-13T18:24:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fe0a0ecd893e49848391d46d9ec485bf, entries=150, sequenceid=165, filesize=11.9 K 2024-11-13T18:24:05,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~59.58 KB/61012, heapSize ~163.13 KB/167040, currentSize=134.18 KB/137400 for debb09c76feaf68052a52c8303bbb905 in 479ms, sequenceid=165, compaction requested=true 2024-11-13T18:24:05,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:05,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:05,163 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:05,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:05,163 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:05,167 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:05,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:05,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:05,167 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/A is initiating minor compaction (all files) 2024-11-13T18:24:05,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:05,167 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/A in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:05,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:05,168 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/698cbd2877a14ffc9e59b97b97bfcd5f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/6272981d9d62439f911a9a9fc0bb2588, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2af99cd069fb472493c16ef1f359f387] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=37.8 K 2024-11-13T18:24:05,175 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 698cbd2877a14ffc9e59b97b97bfcd5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522242492 2024-11-13T18:24:05,177 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:05,177 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/B is initiating minor compaction (all files) 2024-11-13T18:24:05,178 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/B in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:05,178 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/07851e8e62884969a2f7ce22f34af570, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/a4e6556441de4d5a96f9616e07d1e3fa, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/efc16c7f436848f68e0e2044a5ca770b] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=35.5 K 2024-11-13T18:24:05,180 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6272981d9d62439f911a9a9fc0bb2588, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731522244132 2024-11-13T18:24:05,181 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 07851e8e62884969a2f7ce22f34af570, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522242492 2024-11-13T18:24:05,183 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2af99cd069fb472493c16ef1f359f387, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731522244631 2024-11-13T18:24:05,184 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting a4e6556441de4d5a96f9616e07d1e3fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731522244132 2024-11-13T18:24:05,186 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting efc16c7f436848f68e0e2044a5ca770b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731522244631 2024-11-13T18:24:05,211 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:05,212 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/9ce285235dd54bcba5b7aaa5c9d4f64c is 50, key is test_row_0/A:col10/1731522244641/Put/seqid=0 2024-11-13T18:24:05,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:24:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:24:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:05,218 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T18:24:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:05,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:05,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:05,226 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:05,227 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/be18beca70f24c169df66b0c05992041 is 50, key is test_row_0/B:col10/1731522244641/Put/seqid=0 2024-11-13T18:24:05,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/5ecf9e69d28c41f6af1d534e943c14de is 50, key is test_row_0/A:col10/1731522244959/Put/seqid=0 2024-11-13T18:24:05,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741864_1040 (size=12459) 2024-11-13T18:24:05,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741865_1041 (size=12459) 2024-11-13T18:24:05,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741866_1042 (size=12151) 2024-11-13T18:24:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:05,351 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/9ce285235dd54bcba5b7aaa5c9d4f64c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/9ce285235dd54bcba5b7aaa5c9d4f64c 2024-11-13T18:24:05,377 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/A of debb09c76feaf68052a52c8303bbb905 into 9ce285235dd54bcba5b7aaa5c9d4f64c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:05,377 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:05,378 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/A, priority=13, startTime=1731522245163; duration=0sec 2024-11-13T18:24:05,379 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:05,379 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:A 2024-11-13T18:24:05,379 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:05,389 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:05,389 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/C is initiating minor compaction (all files) 2024-11-13T18:24:05,389 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/C in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:05,389 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/257400b10abd4d8daeac0e9efc3caf73, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/64d950561bd74a7384dc78a4b3895c97, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fe0a0ecd893e49848391d46d9ec485bf] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=35.5 K 2024-11-13T18:24:05,396 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 257400b10abd4d8daeac0e9efc3caf73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522242492 2024-11-13T18:24:05,400 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d950561bd74a7384dc78a4b3895c97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731522244132 2024-11-13T18:24:05,402 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe0a0ecd893e49848391d46d9ec485bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731522244631 2024-11-13T18:24:05,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522305364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522305398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522305406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522305414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522305423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,478 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:05,479 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/d6fec423b61a4c29aaebc002a7d874d9 is 50, key is test_row_0/C:col10/1731522244641/Put/seqid=0 2024-11-13T18:24:05,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741867_1043 (size=12459) 2024-11-13T18:24:05,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522305526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522305547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522305547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522305559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522305718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,733 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/5ecf9e69d28c41f6af1d534e943c14de 2024-11-13T18:24:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522305745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,770 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/be18beca70f24c169df66b0c05992041 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/be18beca70f24c169df66b0c05992041 2024-11-13T18:24:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/7a8894d44fe74e33963db27ba11b0897 is 50, key is test_row_0/B:col10/1731522244959/Put/seqid=0 2024-11-13T18:24:05,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522305779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,819 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/B of debb09c76feaf68052a52c8303bbb905 into be18beca70f24c169df66b0c05992041(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:05,819 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:05,819 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/B, priority=13, startTime=1731522245163; duration=0sec 2024-11-13T18:24:05,819 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:05,819 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:B 2024-11-13T18:24:05,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522305795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522305796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:05,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741868_1044 (size=12151) 2024-11-13T18:24:05,866 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/7a8894d44fe74e33963db27ba11b0897 2024-11-13T18:24:05,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/3ab484d2d9f641a08bf9d2b4dd005879 is 50, key is test_row_0/C:col10/1731522244959/Put/seqid=0 2024-11-13T18:24:05,960 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/d6fec423b61a4c29aaebc002a7d874d9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/d6fec423b61a4c29aaebc002a7d874d9 2024-11-13T18:24:05,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741869_1045 (size=12151) 2024-11-13T18:24:05,979 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/C of debb09c76feaf68052a52c8303bbb905 into d6fec423b61a4c29aaebc002a7d874d9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:05,980 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:05,980 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/C, priority=13, startTime=1731522245167; duration=0sec 2024-11-13T18:24:05,980 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:05,980 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:C 2024-11-13T18:24:06,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522306076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:06,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522306128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522306154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:06,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522306155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:24:06,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522306262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:06,375 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/3ab484d2d9f641a08bf9d2b4dd005879 2024-11-13T18:24:06,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/5ecf9e69d28c41f6af1d534e943c14de as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5ecf9e69d28c41f6af1d534e943c14de 2024-11-13T18:24:06,433 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5ecf9e69d28c41f6af1d534e943c14de, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T18:24:06,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/7a8894d44fe74e33963db27ba11b0897 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7a8894d44fe74e33963db27ba11b0897 2024-11-13T18:24:06,458 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7a8894d44fe74e33963db27ba11b0897, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T18:24:06,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/3ab484d2d9f641a08bf9d2b4dd005879 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/3ab484d2d9f641a08bf9d2b4dd005879 2024-11-13T18:24:06,495 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/3ab484d2d9f641a08bf9d2b4dd005879, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T18:24:06,498 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for debb09c76feaf68052a52c8303bbb905 in 1279ms, sequenceid=202, compaction requested=false 2024-11-13T18:24:06,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:06,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:06,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T18:24:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T18:24:06,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T18:24:06,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4470 sec 2024-11-13T18:24:06,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.4800 sec 2024-11-13T18:24:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:06,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:24:06,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:06,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:06,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:06,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:06,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:06,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:06,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f5209c9775cc417ab7b8e8f6a2fd2fad is 50, key is test_row_0/A:col10/1731522245403/Put/seqid=0 2024-11-13T18:24:06,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741870_1046 (size=14541) 2024-11-13T18:24:06,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f5209c9775cc417ab7b8e8f6a2fd2fad 2024-11-13T18:24:06,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/5947d3cf3c1249baaa409ca89fc38981 is 50, key is test_row_0/B:col10/1731522245403/Put/seqid=0 2024-11-13T18:24:06,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731522246741, max timestamp=1731522246809Num uniques 859; 2024-11-13T18:24:06,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:24:06,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731522246741, max timestamp=1731522246809Num uniques 859; 2024-11-13T18:24:06,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:24:06,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731522246741, max timestamp=1731522246809Num uniques 859; 2024-11-13T18:24:06,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:24:06,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741871_1047 (size=12151) 2024-11-13T18:24:06,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522306961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:06,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522306965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522306965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522306977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522307097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522307107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522307109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522307126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/5947d3cf3c1249baaa409ca89fc38981 2024-11-13T18:24:07,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/4e995d4ad4e04dcf95d15dbe1877052f is 50, key is test_row_0/C:col10/1731522245403/Put/seqid=0 2024-11-13T18:24:07,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522307335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,357 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in debb09c76feaf68052a52c8303bbb905 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2361 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522307340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522307346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741872_1048 (size=12151) 2024-11-13T18:24:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522307359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522307363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/4e995d4ad4e04dcf95d15dbe1877052f 2024-11-13T18:24:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f5209c9775cc417ab7b8e8f6a2fd2fad as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f5209c9775cc417ab7b8e8f6a2fd2fad 2024-11-13T18:24:07,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f5209c9775cc417ab7b8e8f6a2fd2fad, entries=200, sequenceid=217, filesize=14.2 K 2024-11-13T18:24:07,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/5947d3cf3c1249baaa409ca89fc38981 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/5947d3cf3c1249baaa409ca89fc38981 2024-11-13T18:24:07,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/5947d3cf3c1249baaa409ca89fc38981, entries=150, sequenceid=217, filesize=11.9 K 2024-11-13T18:24:07,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/4e995d4ad4e04dcf95d15dbe1877052f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/4e995d4ad4e04dcf95d15dbe1877052f 2024-11-13T18:24:07,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/4e995d4ad4e04dcf95d15dbe1877052f, entries=150, sequenceid=217, filesize=11.9 K 2024-11-13T18:24:07,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for debb09c76feaf68052a52c8303bbb905 in 863ms, sequenceid=217, compaction requested=true 2024-11-13T18:24:07,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:07,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:07,492 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:24:07,492 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:07,495 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:07,495 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/A is initiating minor compaction (all files) 2024-11-13T18:24:07,495 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:07,496 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/B is initiating minor compaction (all files) 2024-11-13T18:24:07,495 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/A in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:07,496 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/B in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:07,496 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/be18beca70f24c169df66b0c05992041, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7a8894d44fe74e33963db27ba11b0897, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/5947d3cf3c1249baaa409ca89fc38981] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=35.9 K 2024-11-13T18:24:07,496 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/9ce285235dd54bcba5b7aaa5c9d4f64c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5ecf9e69d28c41f6af1d534e943c14de, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f5209c9775cc417ab7b8e8f6a2fd2fad] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=38.2 K 2024-11-13T18:24:07,497 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting be18beca70f24c169df66b0c05992041, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731522244631 2024-11-13T18:24:07,497 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ce285235dd54bcba5b7aaa5c9d4f64c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731522244631 2024-11-13T18:24:07,498 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a8894d44fe74e33963db27ba11b0897, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522244959 2024-11-13T18:24:07,498 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ecf9e69d28c41f6af1d534e943c14de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522244959 2024-11-13T18:24:07,499 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5209c9775cc417ab7b8e8f6a2fd2fad, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731522245387 2024-11-13T18:24:07,499 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 5947d3cf3c1249baaa409ca89fc38981, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731522245387 2024-11-13T18:24:07,530 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:07,531 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/1df67042e76a40d988ca603fcc29f9ca is 50, key is test_row_0/B:col10/1731522245403/Put/seqid=0 2024-11-13T18:24:07,545 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#A#compaction#37 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:07,546 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/82a439a675a54e5d88f654e1f0d63983 is 50, key is test_row_0/A:col10/1731522245403/Put/seqid=0 2024-11-13T18:24:07,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741873_1049 (size=12561) 2024-11-13T18:24:07,613 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/1df67042e76a40d988ca603fcc29f9ca as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/1df67042e76a40d988ca603fcc29f9ca 2024-11-13T18:24:07,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741874_1050 (size=12561) 2024-11-13T18:24:07,639 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/82a439a675a54e5d88f654e1f0d63983 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/82a439a675a54e5d88f654e1f0d63983 2024-11-13T18:24:07,668 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/A of debb09c76feaf68052a52c8303bbb905 into 82a439a675a54e5d88f654e1f0d63983(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:07,668 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:07,668 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/A, priority=13, startTime=1731522247491; duration=0sec 2024-11-13T18:24:07,669 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:07,669 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:A 2024-11-13T18:24:07,669 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:07,673 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/B of debb09c76feaf68052a52c8303bbb905 into 1df67042e76a40d988ca603fcc29f9ca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:07,673 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:07,673 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/B, priority=13, startTime=1731522247492; duration=0sec 2024-11-13T18:24:07,674 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:07,674 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:B 2024-11-13T18:24:07,675 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:07,675 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/C is initiating minor compaction (all files) 2024-11-13T18:24:07,675 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/C in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:07,675 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/d6fec423b61a4c29aaebc002a7d874d9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/3ab484d2d9f641a08bf9d2b4dd005879, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/4e995d4ad4e04dcf95d15dbe1877052f] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=35.9 K 2024-11-13T18:24:07,677 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6fec423b61a4c29aaebc002a7d874d9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731522244631 2024-11-13T18:24:07,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-13T18:24:07,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:07,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:07,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:07,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:07,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:07,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:07,691 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ab484d2d9f641a08bf9d2b4dd005879, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522244959 2024-11-13T18:24:07,694 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e995d4ad4e04dcf95d15dbe1877052f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731522245387 2024-11-13T18:24:07,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f0ea427823fa451c8004a873d0c7bdf3 is 50, key is test_row_0/A:col10/1731522246960/Put/seqid=0 2024-11-13T18:24:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522307726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522307720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,749 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:07,750 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/b92252f5d9ec4110b2de118aaf8527c4 is 50, key is test_row_0/C:col10/1731522245403/Put/seqid=0 2024-11-13T18:24:07,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522307745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522307745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741875_1051 (size=14541) 2024-11-13T18:24:07,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f0ea427823fa451c8004a873d0c7bdf3 2024-11-13T18:24:07,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741876_1052 (size=12561) 2024-11-13T18:24:07,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/7db7851e6bcc469889b88ac371f12fc6 is 50, key is test_row_0/B:col10/1731522246960/Put/seqid=0 2024-11-13T18:24:07,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522307857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741877_1053 (size=12151) 2024-11-13T18:24:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522307860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/7db7851e6bcc469889b88ac371f12fc6 2024-11-13T18:24:07,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522307860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522307868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:07,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:07,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/62cb77324afa40c68c1bfd5808da2c5a is 50, key is test_row_0/C:col10/1731522246960/Put/seqid=0 2024-11-13T18:24:07,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741878_1054 (size=12151) 2024-11-13T18:24:07,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/62cb77324afa40c68c1bfd5808da2c5a 2024-11-13T18:24:07,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f0ea427823fa451c8004a873d0c7bdf3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f0ea427823fa451c8004a873d0c7bdf3 2024-11-13T18:24:08,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f0ea427823fa451c8004a873d0c7bdf3, entries=200, sequenceid=253, filesize=14.2 K 2024-11-13T18:24:08,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/7db7851e6bcc469889b88ac371f12fc6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7db7851e6bcc469889b88ac371f12fc6 2024-11-13T18:24:08,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7db7851e6bcc469889b88ac371f12fc6, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T18:24:08,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/62cb77324afa40c68c1bfd5808da2c5a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/62cb77324afa40c68c1bfd5808da2c5a 2024-11-13T18:24:08,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522308071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522308079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522308088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522308091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/62cb77324afa40c68c1bfd5808da2c5a, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T18:24:08,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for debb09c76feaf68052a52c8303bbb905 in 435ms, sequenceid=253, compaction requested=false 2024-11-13T18:24:08,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:08,228 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/b92252f5d9ec4110b2de118aaf8527c4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b92252f5d9ec4110b2de118aaf8527c4 2024-11-13T18:24:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:24:08,248 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:08,248 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/C of debb09c76feaf68052a52c8303bbb905 into b92252f5d9ec4110b2de118aaf8527c4(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:08,248 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:08,248 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/C, priority=13, startTime=1731522247492; duration=0sec 2024-11-13T18:24:08,249 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:08,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:08,249 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:C 2024-11-13T18:24:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-13T18:24:08,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:24:08,322 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:08,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:24:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:08,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:24:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:08,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/1ae52dc5af1e42fc9788d6a5ffdf8903 is 50, key is test_row_0/A:col10/1731522248431/Put/seqid=0 2024-11-13T18:24:08,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:08,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:08,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:08,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:08,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741879_1055 (size=12301) 2024-11-13T18:24:08,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522308573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522308593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522308600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522308609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:24:08,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:08,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522308706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522308731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522308731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522308736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:08,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:08,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:08,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:08,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522308918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/1ae52dc5af1e42fc9788d6a5ffdf8903 2024-11-13T18:24:08,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522308947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522308947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:24:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522308967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:08,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/9cfc6207f9aa4ac7bff2285ba3d4bd0d is 50, key is test_row_0/B:col10/1731522248431/Put/seqid=0 2024-11-13T18:24:08,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:08,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:08,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:08,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:08,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741880_1056 (size=12301) 2024-11-13T18:24:09,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/9cfc6207f9aa4ac7bff2285ba3d4bd0d 2024-11-13T18:24:09,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/905abe6c3c2e4754a7c006fdc4614def is 50, key is test_row_0/C:col10/1731522248431/Put/seqid=0 2024-11-13T18:24:09,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741881_1057 (size=12301) 2024-11-13T18:24:09,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/905abe6c3c2e4754a7c006fdc4614def 2024-11-13T18:24:09,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/1ae52dc5af1e42fc9788d6a5ffdf8903 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/1ae52dc5af1e42fc9788d6a5ffdf8903 2024-11-13T18:24:09,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/1ae52dc5af1e42fc9788d6a5ffdf8903, entries=150, sequenceid=268, filesize=12.0 K 2024-11-13T18:24:09,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/9cfc6207f9aa4ac7bff2285ba3d4bd0d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/9cfc6207f9aa4ac7bff2285ba3d4bd0d 2024-11-13T18:24:09,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/9cfc6207f9aa4ac7bff2285ba3d4bd0d, entries=150, sequenceid=268, filesize=12.0 K 2024-11-13T18:24:09,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/905abe6c3c2e4754a7c006fdc4614def as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/905abe6c3c2e4754a7c006fdc4614def 2024-11-13T18:24:09,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:09,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:09,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/905abe6c3c2e4754a7c006fdc4614def, entries=150, sequenceid=268, filesize=12.0 K 2024-11-13T18:24:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for debb09c76feaf68052a52c8303bbb905 in 722ms, sequenceid=268, compaction requested=true 2024-11-13T18:24:09,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:09,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:09,163 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:09,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:09,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:09,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:09,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:09,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:24:09,163 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:09,168 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:09,168 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/A is initiating minor compaction (all files) 2024-11-13T18:24:09,168 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/A in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,168 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/82a439a675a54e5d88f654e1f0d63983, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f0ea427823fa451c8004a873d0c7bdf3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/1ae52dc5af1e42fc9788d6a5ffdf8903] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=38.5 K 2024-11-13T18:24:09,170 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:09,170 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/B is initiating minor compaction (all files) 2024-11-13T18:24:09,170 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/B in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,170 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/1df67042e76a40d988ca603fcc29f9ca, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7db7851e6bcc469889b88ac371f12fc6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/9cfc6207f9aa4ac7bff2285ba3d4bd0d] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=36.1 K 2024-11-13T18:24:09,171 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 1df67042e76a40d988ca603fcc29f9ca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731522245387 2024-11-13T18:24:09,172 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82a439a675a54e5d88f654e1f0d63983, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731522245387 2024-11-13T18:24:09,173 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0ea427823fa451c8004a873d0c7bdf3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731522246960 2024-11-13T18:24:09,173 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db7851e6bcc469889b88ac371f12fc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731522246960 2024-11-13T18:24:09,174 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae52dc5af1e42fc9788d6a5ffdf8903, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731522248423 2024-11-13T18:24:09,174 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cfc6207f9aa4ac7bff2285ba3d4bd0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731522248423 2024-11-13T18:24:09,209 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:09,209 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/f780e71e3ee1421dbf39676a613f9721 is 50, key is test_row_0/B:col10/1731522248431/Put/seqid=0 2024-11-13T18:24:09,211 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:09,232 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f9d6f38df01b48a094b725b2c3beeca7 is 50, key is test_row_0/A:col10/1731522248431/Put/seqid=0 2024-11-13T18:24:09,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:24:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:09,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741882_1058 (size=12813) 2024-11-13T18:24:09,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/0cb51084298b493db9235d494fc3bf16 is 50, key is test_row_0/A:col10/1731522248592/Put/seqid=0 2024-11-13T18:24:09,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522309294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522309310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522309312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522309315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,329 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/f780e71e3ee1421dbf39676a613f9721 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/f780e71e3ee1421dbf39676a613f9721 2024-11-13T18:24:09,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741883_1059 (size=12813) 2024-11-13T18:24:09,348 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/B of debb09c76feaf68052a52c8303bbb905 into f780e71e3ee1421dbf39676a613f9721(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:09,348 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:09,348 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/B, priority=13, startTime=1731522249163; duration=0sec 2024-11-13T18:24:09,348 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:09,348 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:B 2024-11-13T18:24:09,348 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:09,352 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:09,352 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/C is initiating minor compaction (all files) 2024-11-13T18:24:09,352 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/C in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,352 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b92252f5d9ec4110b2de118aaf8527c4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/62cb77324afa40c68c1bfd5808da2c5a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/905abe6c3c2e4754a7c006fdc4614def] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=36.1 K 2024-11-13T18:24:09,353 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting b92252f5d9ec4110b2de118aaf8527c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731522245387 2024-11-13T18:24:09,357 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 62cb77324afa40c68c1bfd5808da2c5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731522246960 2024-11-13T18:24:09,360 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 905abe6c3c2e4754a7c006fdc4614def, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731522248423 2024-11-13T18:24:09,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741884_1060 (size=14741) 2024-11-13T18:24:09,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/0cb51084298b493db9235d494fc3bf16 2024-11-13T18:24:09,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522309379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/84a2259d56c546bd8bc58b496f4875ff is 50, key is test_row_0/B:col10/1731522248592/Put/seqid=0 2024-11-13T18:24:09,403 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in debb09c76feaf68052a52c8303bbb905 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4410 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,404 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#C#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:09,405 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/a09a1e47d09c4d36bace2c1eff118844 is 50, key is test_row_0/C:col10/1731522248431/Put/seqid=0 2024-11-13T18:24:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522309428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522309428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522309438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522309428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741886_1062 (size=12813) 2024-11-13T18:24:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:24:09,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:09,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:09,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741885_1061 (size=12301) 2024-11-13T18:24:09,519 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/a09a1e47d09c4d36bace2c1eff118844 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/a09a1e47d09c4d36bace2c1eff118844 2024-11-13T18:24:09,594 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/C of debb09c76feaf68052a52c8303bbb905 into a09a1e47d09c4d36bace2c1eff118844(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:09,595 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:09,595 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/C, priority=13, startTime=1731522249163; duration=0sec 2024-11-13T18:24:09,598 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:09,598 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:C 2024-11-13T18:24:09,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:09,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:09,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522309657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522309657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522309663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522309663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:09,691 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:24:09,815 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f9d6f38df01b48a094b725b2c3beeca7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f9d6f38df01b48a094b725b2c3beeca7 2024-11-13T18:24:09,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:09,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:09,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:09,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:09,879 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/A of debb09c76feaf68052a52c8303bbb905 into f9d6f38df01b48a094b725b2c3beeca7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:09,879 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:09,879 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/A, priority=13, startTime=1731522249163; duration=0sec 2024-11-13T18:24:09,879 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:09,879 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:A 2024-11-13T18:24:09,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/84a2259d56c546bd8bc58b496f4875ff 2024-11-13T18:24:09,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/b17b881ab1d14acba926db993957219f is 50, key is test_row_0/C:col10/1731522248592/Put/seqid=0 2024-11-13T18:24:09,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522309979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:09,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:09,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:10,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:10,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:10,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:10,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:10,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:10,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522309995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522309999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522309996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741887_1063 (size=12301) 2024-11-13T18:24:10,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/b17b881ab1d14acba926db993957219f 2024-11-13T18:24:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/0cb51084298b493db9235d494fc3bf16 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/0cb51084298b493db9235d494fc3bf16 2024-11-13T18:24:10,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/0cb51084298b493db9235d494fc3bf16, entries=200, sequenceid=288, filesize=14.4 K 2024-11-13T18:24:10,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/84a2259d56c546bd8bc58b496f4875ff as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/84a2259d56c546bd8bc58b496f4875ff 2024-11-13T18:24:10,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/84a2259d56c546bd8bc58b496f4875ff, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T18:24:10,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/b17b881ab1d14acba926db993957219f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b17b881ab1d14acba926db993957219f 2024-11-13T18:24:10,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b17b881ab1d14acba926db993957219f, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T18:24:10,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for debb09c76feaf68052a52c8303bbb905 in 853ms, sequenceid=288, compaction requested=false 2024-11-13T18:24:10,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:10,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:24:10,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:10,172 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:24:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/44b2b1b08db7437282410688145d90c3 is 50, key is test_row_0/A:col10/1731522249290/Put/seqid=0 2024-11-13T18:24:10,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741888_1064 (size=12301) 2024-11-13T18:24:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:24:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:10,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:10,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522310601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522310602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522310607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522310616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,694 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/44b2b1b08db7437282410688145d90c3 2024-11-13T18:24:10,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522310719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/647d8f63200845e3af5447ece688f2a7 is 50, key is test_row_0/B:col10/1731522249290/Put/seqid=0 2024-11-13T18:24:10,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522310727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522310728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522310728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741889_1065 (size=12301) 2024-11-13T18:24:10,801 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/647d8f63200845e3af5447ece688f2a7 2024-11-13T18:24:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/b0a19097ad4a4b1fa366cd61c9a60444 is 50, key is test_row_0/C:col10/1731522249290/Put/seqid=0 2024-11-13T18:24:10,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741890_1066 (size=12301) 2024-11-13T18:24:10,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522310926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,938 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/b0a19097ad4a4b1fa366cd61c9a60444 2024-11-13T18:24:10,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522310959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522310959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522310967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:10,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/44b2b1b08db7437282410688145d90c3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/44b2b1b08db7437282410688145d90c3 2024-11-13T18:24:11,017 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/44b2b1b08db7437282410688145d90c3, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T18:24:11,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/647d8f63200845e3af5447ece688f2a7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/647d8f63200845e3af5447ece688f2a7 2024-11-13T18:24:11,081 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/647d8f63200845e3af5447ece688f2a7, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T18:24:11,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/b0a19097ad4a4b1fa366cd61c9a60444 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b0a19097ad4a4b1fa366cd61c9a60444 2024-11-13T18:24:11,117 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b0a19097ad4a4b1fa366cd61c9a60444, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T18:24:11,120 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for debb09c76feaf68052a52c8303bbb905 in 947ms, sequenceid=303, compaction requested=true 2024-11-13T18:24:11,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:11,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:11,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T18:24:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T18:24:11,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T18:24:11,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8010 sec 2024-11-13T18:24:11,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.8280 sec 2024-11-13T18:24:11,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522250592, max timestamp=1731522251261Num uniques 859; 2024-11-13T18:24:11,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:11,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-13T18:24:11,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522250592, max timestamp=1731522251261Num uniques 859; 2024-11-13T18:24:11,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522250592, max timestamp=1731522251261Num uniques 859; 2024-11-13T18:24:11,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:11,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:11,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:11,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:11,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:11,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:11,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:11,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:11,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/5186fcea39974af3b77b37644ca7fec2 is 50, key is test_row_0/A:col10/1731522250592/Put/seqid=0 2024-11-13T18:24:11,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741891_1067 (size=12301) 2024-11-13T18:24:11,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522251365, max timestamp=1731522251420Num uniques 859; 2024-11-13T18:24:11,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:24:11,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522251365, max timestamp=1731522251420Num uniques 859; 2024-11-13T18:24:11,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:24:11,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522251365, max timestamp=1731522251420Num uniques 859; 2024-11-13T18:24:11,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:24:11,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731522251480, max timestamp=1731522251662Num uniques 1068; 2024-11-13T18:24:11,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T18:24:11,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731522251480, max timestamp=1731522251662Num uniques 1068; 2024-11-13T18:24:11,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T18:24:11,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731522251480, max timestamp=1731522251662Num uniques 1068; 2024-11-13T18:24:11,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T18:24:11,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/5186fcea39974af3b77b37644ca7fec2 2024-11-13T18:24:11,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/57247da1b89b4dd2ba72c7259946d57a is 50, key is test_row_0/B:col10/1731522250592/Put/seqid=0 2024-11-13T18:24:11,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731522251707, max timestamp=1731522251801Num uniques 1077; 2024-11-13T18:24:11,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-13T18:24:11,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731522251707, max timestamp=1731522251801Num uniques 1077; 2024-11-13T18:24:11,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-13T18:24:11,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522251775, max timestamp=1731522251801Num uniques 1077; 2024-11-13T18:24:11,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=6870, suffix heap size=139976,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=150 2024-11-13T18:24:11,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741892_1068 (size=12301) 2024-11-13T18:24:11,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731522251908, max timestamp=1731522251931Num uniques 1136; 2024-11-13T18:24:11,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-13T18:24:11,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731522251908, max timestamp=1731522251931Num uniques 1009; 2024-11-13T18:24:11,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13096, suffix heap size=132904,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=286 2024-11-13T18:24:11,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731522251908, max timestamp=1731522251931Num uniques 1136; 2024-11-13T18:24:11,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-13T18:24:12,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731522252019, max timestamp=1731522252101Num uniques 1145; 2024-11-13T18:24:12,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-13T18:24:12,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731522252019, max timestamp=1731522252101Num uniques 1145; 2024-11-13T18:24:12,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-13T18:24:12,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731522252019, max timestamp=1731522252101Num uniques 1145; 2024-11-13T18:24:12,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-13T18:24:12,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731522252157, max timestamp=1731522252213Num uniques 1054; 2024-11-13T18:24:12,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731522252157, max timestamp=1731522252213Num uniques 1054; 2024-11-13T18:24:12,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731522252157, max timestamp=1731522252213Num uniques 1054; 2024-11-13T18:24:12,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-13T18:24:12,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-13T18:24:12,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-13T18:24:12,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/57247da1b89b4dd2ba72c7259946d57a 2024-11-13T18:24:12,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/f5565aacc9b94f63aa8c694c6bca82d4 is 50, key is test_row_0/C:col10/1731522250592/Put/seqid=0 2024-11-13T18:24:12,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741893_1069 (size=12301) 2024-11-13T18:24:12,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/f5565aacc9b94f63aa8c694c6bca82d4 2024-11-13T18:24:12,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731522252293, max timestamp=1731522252330Num uniques 1113; 2024-11-13T18:24:12,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-13T18:24:12,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731522252293, max timestamp=1731522252330Num uniques 1113; 2024-11-13T18:24:12,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-13T18:24:12,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731522252293, max timestamp=1731522252330Num uniques 1113; 2024-11-13T18:24:12,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-13T18:24:12,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/5186fcea39974af3b77b37644ca7fec2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5186fcea39974af3b77b37644ca7fec2 2024-11-13T18:24:12,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5186fcea39974af3b77b37644ca7fec2, entries=150, sequenceid=324, filesize=12.0 K 2024-11-13T18:24:12,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/57247da1b89b4dd2ba72c7259946d57a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/57247da1b89b4dd2ba72c7259946d57a 2024-11-13T18:24:12,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/57247da1b89b4dd2ba72c7259946d57a, entries=150, sequenceid=324, filesize=12.0 K 2024-11-13T18:24:12,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/f5565aacc9b94f63aa8c694c6bca82d4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f5565aacc9b94f63aa8c694c6bca82d4 2024-11-13T18:24:12,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f5565aacc9b94f63aa8c694c6bca82d4, entries=150, sequenceid=324, filesize=12.0 K 2024-11-13T18:24:12,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=145.36 KB/148850 for debb09c76feaf68052a52c8303bbb905 in 1210ms, sequenceid=324, compaction requested=true 2024-11-13T18:24:12,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:12,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:12,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:12,481 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:24:12,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:12,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:12,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:12,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:24:12,482 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:24:12,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731522252446, max timestamp=1731522252480Num uniques 1122; 2024-11-13T18:24:12,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-13T18:24:12,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731522252446, max timestamp=1731522252480Num uniques 1122; 2024-11-13T18:24:12,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-13T18:24:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:12,490 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:24:12,490 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52156 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:24:12,490 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/B is initiating minor compaction (all files) 2024-11-13T18:24:12,490 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/A is initiating minor compaction (all files) 2024-11-13T18:24:12,490 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/A in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:12,490 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f9d6f38df01b48a094b725b2c3beeca7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/0cb51084298b493db9235d494fc3bf16, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/44b2b1b08db7437282410688145d90c3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5186fcea39974af3b77b37644ca7fec2] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=50.9 K 2024-11-13T18:24:12,491 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/B in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:12,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=73.80 KB heapSize=203.42 KB 2024-11-13T18:24:12,491 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/f780e71e3ee1421dbf39676a613f9721, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/84a2259d56c546bd8bc58b496f4875ff, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/647d8f63200845e3af5447ece688f2a7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/57247da1b89b4dd2ba72c7259946d57a] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=48.6 K 2024-11-13T18:24:12,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:12,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:12,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:12,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:12,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:12,491 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting f780e71e3ee1421dbf39676a613f9721, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731522248423 2024-11-13T18:24:12,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731522252446, max timestamp=1731522252480Num uniques 1072; 2024-11-13T18:24:12,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-13T18:24:12,492 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting f9d6f38df01b48a094b725b2c3beeca7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731522248423 2024-11-13T18:24:12,492 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84a2259d56c546bd8bc58b496f4875ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731522248586 2024-11-13T18:24:12,493 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cb51084298b493db9235d494fc3bf16, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731522248586 2024-11-13T18:24:12,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:12,499 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 647d8f63200845e3af5447ece688f2a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731522249290 2024-11-13T18:24:12,499 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 44b2b1b08db7437282410688145d90c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731522249290 2024-11-13T18:24:12,503 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 5186fcea39974af3b77b37644ca7fec2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731522250592 2024-11-13T18:24:12,504 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57247da1b89b4dd2ba72c7259946d57a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731522250592 2024-11-13T18:24:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:24:12,508 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:12,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:12,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/e8db7227119948ffa3e7ca15c1489065 is 50, key is test_row_0/A:col10/1731522252488/Put/seqid=0 2024-11-13T18:24:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-13T18:24:12,518 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:12,520 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:12,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:24:12,552 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#B#compaction#58 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:12,553 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/fc8af82c7a8846bfb363038aaf6032ab is 50, key is test_row_0/B:col10/1731522250592/Put/seqid=0 2024-11-13T18:24:12,557 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#A#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:12,558 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/5a5b21aef69d4917b488c731dd443da4 is 50, key is test_row_0/A:col10/1731522250592/Put/seqid=0 2024-11-13T18:24:12,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741894_1070 (size=12301) 2024-11-13T18:24:12,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/e8db7227119948ffa3e7ca15c1489065 2024-11-13T18:24:12,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741895_1071 (size=12949) 2024-11-13T18:24:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:24:12,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741896_1072 (size=12949) 2024-11-13T18:24:12,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/212f488ff416473596c5873679c4653b is 50, key is test_row_0/B:col10/1731522252488/Put/seqid=0 2024-11-13T18:24:12,666 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/fc8af82c7a8846bfb363038aaf6032ab as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/fc8af82c7a8846bfb363038aaf6032ab 2024-11-13T18:24:12,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522252574, max timestamp=1731522252663Num uniques 859; 2024-11-13T18:24:12,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:12,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522252574, max timestamp=1731522252665Num uniques 859; 2024-11-13T18:24:12,675 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/5a5b21aef69d4917b488c731dd443da4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5a5b21aef69d4917b488c731dd443da4 2024-11-13T18:24:12,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522252574, max timestamp=1731522252665Num uniques 859; 2024-11-13T18:24:12,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:24:12,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:24:12,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:12,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:12,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:12,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:12,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:12,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:12,692 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in debb09c76feaf68052a52c8303bbb905/B of debb09c76feaf68052a52c8303bbb905 into fc8af82c7a8846bfb363038aaf6032ab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:12,692 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:12,692 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/B, priority=12, startTime=1731522252481; duration=0sec 2024-11-13T18:24:12,692 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:12,692 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:B 2024-11-13T18:24:12,693 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:24:12,695 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:24:12,695 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/C is initiating minor compaction (all files) 2024-11-13T18:24:12,695 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/C in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:12,696 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/a09a1e47d09c4d36bace2c1eff118844, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b17b881ab1d14acba926db993957219f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b0a19097ad4a4b1fa366cd61c9a60444, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f5565aacc9b94f63aa8c694c6bca82d4] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=48.6 K 2024-11-13T18:24:12,697 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting a09a1e47d09c4d36bace2c1eff118844, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731522248423 2024-11-13T18:24:12,698 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b17b881ab1d14acba926db993957219f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731522248586 2024-11-13T18:24:12,698 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0a19097ad4a4b1fa366cd61c9a60444, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731522249290 2024-11-13T18:24:12,699 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5565aacc9b94f63aa8c694c6bca82d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731522250592 2024-11-13T18:24:12,703 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in debb09c76feaf68052a52c8303bbb905/A of debb09c76feaf68052a52c8303bbb905 into 5a5b21aef69d4917b488c731dd443da4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:12,703 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:12,703 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/A, priority=12, startTime=1731522252481; duration=0sec 2024-11-13T18:24:12,704 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:12,704 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:A 2024-11-13T18:24:12,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741897_1073 (size=12301) 2024-11-13T18:24:12,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/212f488ff416473596c5873679c4653b 2024-11-13T18:24:12,733 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:12,734 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/b090e7f1a9544bd48c8221ef0dbb44a8 is 50, key is test_row_0/C:col10/1731522250592/Put/seqid=0 2024-11-13T18:24:12,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/9bf6a97debcb4e2baed2e5f02f027cd8 is 50, key is test_row_0/C:col10/1731522252488/Put/seqid=0 2024-11-13T18:24:12,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741898_1074 (size=12949) 2024-11-13T18:24:12,789 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/b090e7f1a9544bd48c8221ef0dbb44a8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b090e7f1a9544bd48c8221ef0dbb44a8 2024-11-13T18:24:12,799 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in debb09c76feaf68052a52c8303bbb905/C of debb09c76feaf68052a52c8303bbb905 into b090e7f1a9544bd48c8221ef0dbb44a8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:12,800 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:12,800 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/C, priority=12, startTime=1731522252481; duration=0sec 2024-11-13T18:24:12,800 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:12,800 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:C 2024-11-13T18:24:12,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741899_1075 (size=12301) 2024-11-13T18:24:12,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731522252760, max timestamp=1731522252815Num uniques 1018; 2024-11-13T18:24:12,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-13T18:24:12,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731522252760, max timestamp=1731522252815Num uniques 1018; 2024-11-13T18:24:12,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731522252760, max timestamp=1731522252815Num uniques 1009; 2024-11-13T18:24:12,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-13T18:24:12,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-11-13T18:24:12,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:24:12,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:12,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:12,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:12,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:12,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:24:12,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731522252901, max timestamp=1731522252952Num uniques 1127; 2024-11-13T18:24:12,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731522252901, max timestamp=1731522252952Num uniques 1127; 2024-11-13T18:24:12,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-13T18:24:12,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-13T18:24:12,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731522252901, max timestamp=1731522252952Num uniques 1127; 2024-11-13T18:24:12,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-13T18:24:13,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:24:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13086, totalHeapSize=29984, min timestamp=1731522253035, max timestamp=1731522253068Num uniques 1086; 2024-11-13T18:24:13,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731522253035, max timestamp=1731522253068Num uniques 1086; 2024-11-13T18:24:13,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-13T18:24:13,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=13086, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-13T18:24:13,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731522253040, max timestamp=1731522253067Num uniques 1086; 2024-11-13T18:24:13,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-13T18:24:13,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:24:13,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731522253129, max timestamp=1731522253156Num uniques 1145; 2024-11-13T18:24:13,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731522253129, max timestamp=1731522253156Num uniques 1145; 2024-11-13T18:24:13,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-13T18:24:13,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-13T18:24:13,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:24:13,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:13,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731522253129, max timestamp=1731522253156Num uniques 1045; 2024-11-13T18:24:13,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-13T18:24:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/9bf6a97debcb4e2baed2e5f02f027cd8 2024-11-13T18:24:13,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/e8db7227119948ffa3e7ca15c1489065 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/e8db7227119948ffa3e7ca15c1489065 2024-11-13T18:24:13,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/e8db7227119948ffa3e7ca15c1489065, entries=150, sequenceid=466, filesize=12.0 K 2024-11-13T18:24:13,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/212f488ff416473596c5873679c4653b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/212f488ff416473596c5873679c4653b 2024-11-13T18:24:13,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/212f488ff416473596c5873679c4653b, entries=150, sequenceid=466, filesize=12.0 K 2024-11-13T18:24:13,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/9bf6a97debcb4e2baed2e5f02f027cd8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/9bf6a97debcb4e2baed2e5f02f027cd8 2024-11-13T18:24:13,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/9bf6a97debcb4e2baed2e5f02f027cd8, entries=150, sequenceid=466, filesize=12.0 K 2024-11-13T18:24:13,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731522253258, max timestamp=1731522253281Num uniques 1054; 2024-11-13T18:24:13,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731522253258, max timestamp=1731522253281Num uniques 1054; 2024-11-13T18:24:13,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-13T18:24:13,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-13T18:24:13,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~84.89 KB/86928, currentSize=140.89 KB/144270 for debb09c76feaf68052a52c8303bbb905 in 793ms, sequenceid=466, compaction requested=false 2024-11-13T18:24:13,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:13,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731522253258, max timestamp=1731522253281Num uniques 1054; 2024-11-13T18:24:13,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-13T18:24:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:13,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=53.67 KB heapSize=146.39 KB 2024-11-13T18:24:13,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:13,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:13,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:13,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:13,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:13,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:13,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/cead70067a8f4aa0b50ed86c519eb2bd is 50, key is test_row_0/A:col10/1731522253265/Put/seqid=0 2024-11-13T18:24:13,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:24:13,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:13,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741900_1076 (size=24505) 2024-11-13T18:24:13,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522253343, max timestamp=1731522253398Num uniques 859; 2024-11-13T18:24:13,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:13,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522253343, max timestamp=1731522253398Num uniques 859; 2024-11-13T18:24:13,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:13,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522253343, max timestamp=1731522253398Num uniques 859; 2024-11-13T18:24:13,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:13,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:24:13,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:13,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522313510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731522253462, max timestamp=1731522253531Num uniques 1018; 2024-11-13T18:24:13,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-13T18:24:13,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731522253462, max timestamp=1731522253531Num uniques 1018; 2024-11-13T18:24:13,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-13T18:24:13,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731522253462, max timestamp=1731522253531Num uniques 1018; 2024-11-13T18:24:13,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:24:13,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522313620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522313639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522313646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522313648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522313652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:24:13,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:24:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522313768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522313768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/cead70067a8f4aa0b50ed86c519eb2bd 2024-11-13T18:24:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522313778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522313778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/e464575d7a8d45049a20f1a6ad56841f is 50, key is test_row_0/B:col10/1731522253265/Put/seqid=0 2024-11-13T18:24:13,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:24:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:13,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:13,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:13,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522313871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:13,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741901_1077 (size=12301) 2024-11-13T18:24:13,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/e464575d7a8d45049a20f1a6ad56841f 2024-11-13T18:24:13,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/30e0d4c12b574420a46881e33fc7775e is 50, key is test_row_0/C:col10/1731522253265/Put/seqid=0 2024-11-13T18:24:14,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:24:14,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:14,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:14,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:14,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:14,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:14,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741902_1078 (size=12301) 2024-11-13T18:24:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:14,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/30e0d4c12b574420a46881e33fc7775e 2024-11-13T18:24:14,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522314008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:14,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522314009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:14,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:14,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522314009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:14,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522314019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:14,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/cead70067a8f4aa0b50ed86c519eb2bd as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/cead70067a8f4aa0b50ed86c519eb2bd 2024-11-13T18:24:14,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/cead70067a8f4aa0b50ed86c519eb2bd, entries=400, sequenceid=580, filesize=23.9 K 2024-11-13T18:24:14,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/e464575d7a8d45049a20f1a6ad56841f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e464575d7a8d45049a20f1a6ad56841f 2024-11-13T18:24:14,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e464575d7a8d45049a20f1a6ad56841f, entries=150, sequenceid=580, filesize=12.0 K 2024-11-13T18:24:14,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/30e0d4c12b574420a46881e33fc7775e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/30e0d4c12b574420a46881e33fc7775e 2024-11-13T18:24:14,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/30e0d4c12b574420a46881e33fc7775e, entries=150, sequenceid=580, filesize=12.0 K 2024-11-13T18:24:14,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~146.34 KB/149856, currentSize=134.18 KB/137400 for debb09c76feaf68052a52c8303bbb905 in 838ms, sequenceid=580, compaction requested=true 2024-11-13T18:24:14,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:14,147 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:14,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:14,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:14,148 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:14,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:14,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:14,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:14,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:14,157 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:14,157 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/B is initiating minor compaction (all files) 2024-11-13T18:24:14,157 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/B in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:14,157 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/fc8af82c7a8846bfb363038aaf6032ab, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/212f488ff416473596c5873679c4653b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e464575d7a8d45049a20f1a6ad56841f] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=36.7 K 2024-11-13T18:24:14,159 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 49755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:14,160 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/A is initiating minor compaction (all files) 2024-11-13T18:24:14,160 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/A in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:14,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:24:14,160 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5a5b21aef69d4917b488c731dd443da4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/e8db7227119948ffa3e7ca15c1489065, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/cead70067a8f4aa0b50ed86c519eb2bd] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=48.6 K 2024-11-13T18:24:14,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:14,161 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=134.18 KB heapSize=374.77 KB 2024-11-13T18:24:14,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:14,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:14,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:14,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:14,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:14,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:14,164 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a5b21aef69d4917b488c731dd443da4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731522250592 2024-11-13T18:24:14,164 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting fc8af82c7a8846bfb363038aaf6032ab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731522250592 2024-11-13T18:24:14,165 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 212f488ff416473596c5873679c4653b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731522252467 2024-11-13T18:24:14,166 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8db7227119948ffa3e7ca15c1489065, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731522252467 2024-11-13T18:24:14,171 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e464575d7a8d45049a20f1a6ad56841f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1731522253265 2024-11-13T18:24:14,174 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting cead70067a8f4aa0b50ed86c519eb2bd, keycount=400, bloomtype=ROW, size=23.9 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1731522253258 2024-11-13T18:24:14,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/49618bb9c15d4db0a6549322e63d0cbe is 50, key is test_row_0/A:col10/1731522253646/Put/seqid=0 2024-11-13T18:24:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:14,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:14,212 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:14,213 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/bb50b1280faa405a8d610e34c7b0a3ef is 50, key is test_row_0/A:col10/1731522253265/Put/seqid=0 2024-11-13T18:24:14,221 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#B#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:14,222 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/ee5dbcd683ee4fd7b13a38d0619becd8 is 50, key is test_row_0/B:col10/1731522253265/Put/seqid=0 2024-11-13T18:24:14,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741903_1079 (size=12301) 2024-11-13T18:24:14,280 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/49618bb9c15d4db0a6549322e63d0cbe 2024-11-13T18:24:14,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741904_1080 (size=13051) 2024-11-13T18:24:14,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522314348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:14,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522314348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:14,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522314348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:14,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741905_1081 (size=13051) 2024-11-13T18:24:14,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/3b52874fec8040729e3bc062ec1521ae is 50, key is test_row_0/B:col10/1731522253646/Put/seqid=0 2024-11-13T18:24:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:14,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522314363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:14,440 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/ee5dbcd683ee4fd7b13a38d0619becd8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ee5dbcd683ee4fd7b13a38d0619becd8 2024-11-13T18:24:14,456 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/B of debb09c76feaf68052a52c8303bbb905 into ee5dbcd683ee4fd7b13a38d0619becd8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:14,456 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:14,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:14,456 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/B, priority=13, startTime=1731522254148; duration=0sec 2024-11-13T18:24:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522314361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:14,456 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:14,456 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:B 2024-11-13T18:24:14,456 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:14,458 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:14,458 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/C is initiating minor compaction (all files) 2024-11-13T18:24:14,458 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/C in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:14,459 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b090e7f1a9544bd48c8221ef0dbb44a8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/9bf6a97debcb4e2baed2e5f02f027cd8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/30e0d4c12b574420a46881e33fc7775e] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=36.7 K 2024-11-13T18:24:14,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741906_1082 (size=12301) 2024-11-13T18:24:14,462 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting b090e7f1a9544bd48c8221ef0dbb44a8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731522250592 2024-11-13T18:24:14,462 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/3b52874fec8040729e3bc062ec1521ae 2024-11-13T18:24:14,465 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf6a97debcb4e2baed2e5f02f027cd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731522252467 2024-11-13T18:24:14,466 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 30e0d4c12b574420a46881e33fc7775e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1731522253265 2024-11-13T18:24:14,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/eaedd4fa38bd437f8093aeeae8c9669f is 50, key is test_row_0/C:col10/1731522253646/Put/seqid=0 2024-11-13T18:24:14,512 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:14,513 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/f4c7a2a103b048e29fd89c3f9e00c301 is 50, key is test_row_0/C:col10/1731522253265/Put/seqid=0 2024-11-13T18:24:14,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522314567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:14,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741908_1084 (size=13051) 2024-11-13T18:24:14,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741907_1083 (size=12301) 2024-11-13T18:24:14,594 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/eaedd4fa38bd437f8093aeeae8c9669f 2024-11-13T18:24:14,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/49618bb9c15d4db0a6549322e63d0cbe as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/49618bb9c15d4db0a6549322e63d0cbe 2024-11-13T18:24:14,648 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/f4c7a2a103b048e29fd89c3f9e00c301 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f4c7a2a103b048e29fd89c3f9e00c301 2024-11-13T18:24:14,669 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/49618bb9c15d4db0a6549322e63d0cbe, entries=150, sequenceid=632, filesize=12.0 K 2024-11-13T18:24:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:24:14,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/3b52874fec8040729e3bc062ec1521ae as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/3b52874fec8040729e3bc062ec1521ae 2024-11-13T18:24:14,680 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/C of debb09c76feaf68052a52c8303bbb905 into f4c7a2a103b048e29fd89c3f9e00c301(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:14,681 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:14,681 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/C, priority=13, startTime=1731522254149; duration=0sec 2024-11-13T18:24:14,681 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:14,681 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:C 2024-11-13T18:24:14,694 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/3b52874fec8040729e3bc062ec1521ae, entries=150, sequenceid=632, filesize=12.0 K 2024-11-13T18:24:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/eaedd4fa38bd437f8093aeeae8c9669f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/eaedd4fa38bd437f8093aeeae8c9669f 2024-11-13T18:24:14,718 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/eaedd4fa38bd437f8093aeeae8c9669f, entries=150, sequenceid=632, filesize=12.0 K 2024-11-13T18:24:14,725 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~374.72 KB/383712, currentSize=46.96 KB/48090 for debb09c76feaf68052a52c8303bbb905 in 564ms, sequenceid=632, compaction requested=false 2024-11-13T18:24:14,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:14,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-13T18:24:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-13T18:24:14,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-13T18:24:14,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2110 sec 2024-11-13T18:24:14,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.2340 sec 2024-11-13T18:24:14,780 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/bb50b1280faa405a8d610e34c7b0a3ef as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb50b1280faa405a8d610e34c7b0a3ef 2024-11-13T18:24:14,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:24:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:14,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:14,796 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/A of debb09c76feaf68052a52c8303bbb905 into bb50b1280faa405a8d610e34c7b0a3ef(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:14,796 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:14,796 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/A, priority=13, startTime=1731522254147; duration=0sec 2024-11-13T18:24:14,797 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:14,797 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:A 2024-11-13T18:24:14,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/edca6cc3321a426f87f4da64b6859cf0 is 50, key is test_row_0/A:col10/1731522254261/Put/seqid=0 2024-11-13T18:24:14,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741909_1085 (size=12301) 2024-11-13T18:24:14,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/edca6cc3321a426f87f4da64b6859cf0 2024-11-13T18:24:14,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/ea05e5509ce14a1e8b9a19104484fc28 is 50, key is test_row_0/B:col10/1731522254261/Put/seqid=0 2024-11-13T18:24:14,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741910_1086 (size=12301) 2024-11-13T18:24:14,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731522254917, max timestamp=1731522254962Num uniques 859; 2024-11-13T18:24:14,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-13T18:24:14,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731522254917, max timestamp=1731522254962Num uniques 859; 2024-11-13T18:24:14,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731522254917, max timestamp=1731522254962Num uniques 859; 2024-11-13T18:24:14,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-13T18:24:14,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-13T18:24:15,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522314995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522315024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522315025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522315029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522315031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522315137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522315157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522315167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522315167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522315167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/ea05e5509ce14a1e8b9a19104484fc28 2024-11-13T18:24:15,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522315357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/2efccbd3e27549b99f6636c6463cef10 is 50, key is test_row_0/C:col10/1731522254261/Put/seqid=0 2024-11-13T18:24:15,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522315388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522315406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522315407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522315408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741911_1087 (size=12301) 2024-11-13T18:24:15,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522315699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522315726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522315726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522315725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522315736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:15,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/2efccbd3e27549b99f6636c6463cef10 2024-11-13T18:24:15,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/edca6cc3321a426f87f4da64b6859cf0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/edca6cc3321a426f87f4da64b6859cf0 2024-11-13T18:24:16,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/edca6cc3321a426f87f4da64b6859cf0, entries=150, sequenceid=645, filesize=12.0 K 2024-11-13T18:24:16,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/ea05e5509ce14a1e8b9a19104484fc28 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ea05e5509ce14a1e8b9a19104484fc28 2024-11-13T18:24:16,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ea05e5509ce14a1e8b9a19104484fc28, entries=150, sequenceid=645, filesize=12.0 K 2024-11-13T18:24:16,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/2efccbd3e27549b99f6636c6463cef10 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/2efccbd3e27549b99f6636c6463cef10 2024-11-13T18:24:16,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/2efccbd3e27549b99f6636c6463cef10, entries=150, sequenceid=645, filesize=12.0 K 2024-11-13T18:24:16,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for debb09c76feaf68052a52c8303bbb905 in 1330ms, sequenceid=645, compaction requested=true 2024-11-13T18:24:16,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:24:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:16,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:24:16,123 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:16,124 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:16,148 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:16,148 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/C is initiating minor compaction (all files) 2024-11-13T18:24:16,149 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/C in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:16,149 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f4c7a2a103b048e29fd89c3f9e00c301, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/eaedd4fa38bd437f8093aeeae8c9669f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/2efccbd3e27549b99f6636c6463cef10] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=36.8 K 2024-11-13T18:24:16,150 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:16,150 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/A is initiating minor compaction (all files) 2024-11-13T18:24:16,150 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/A in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:16,150 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb50b1280faa405a8d610e34c7b0a3ef, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/49618bb9c15d4db0a6549322e63d0cbe, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/edca6cc3321a426f87f4da64b6859cf0] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=36.8 K 2024-11-13T18:24:16,151 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting f4c7a2a103b048e29fd89c3f9e00c301, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1731522253265 2024-11-13T18:24:16,152 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb50b1280faa405a8d610e34c7b0a3ef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1731522253265 2024-11-13T18:24:16,152 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting eaedd4fa38bd437f8093aeeae8c9669f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1731522253611 2024-11-13T18:24:16,153 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49618bb9c15d4db0a6549322e63d0cbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1731522253611 2024-11-13T18:24:16,153 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 2efccbd3e27549b99f6636c6463cef10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1731522254243 2024-11-13T18:24:16,154 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting edca6cc3321a426f87f4da64b6859cf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1731522254243 2024-11-13T18:24:16,209 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:16,213 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/3a48b29c5edf49b1a48b144a832128eb is 50, key is test_row_0/A:col10/1731522254261/Put/seqid=0 2024-11-13T18:24:16,233 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#C#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:16,234 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/1eb484c6de9c47ff8dfd21f9f8e43d3d is 50, key is test_row_0/C:col10/1731522254261/Put/seqid=0 2024-11-13T18:24:16,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=147.60 KB heapSize=392.02 KB 2024-11-13T18:24:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:16,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:16,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741912_1088 (size=13153) 2024-11-13T18:24:16,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741913_1089 (size=13153) 2024-11-13T18:24:16,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/bb634d6ad1204100b938a2489f946ef6 is 50, key is test_row_0/A:col10/1731522256267/Put/seqid=0 2024-11-13T18:24:16,316 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/3a48b29c5edf49b1a48b144a832128eb as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/3a48b29c5edf49b1a48b144a832128eb 2024-11-13T18:24:16,329 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/A of debb09c76feaf68052a52c8303bbb905 into 3a48b29c5edf49b1a48b144a832128eb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:16,329 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:16,329 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/A, priority=13, startTime=1731522256123; duration=0sec 2024-11-13T18:24:16,329 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:16,329 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:A 2024-11-13T18:24:16,330 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:16,335 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:16,335 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/B is initiating minor compaction (all files) 2024-11-13T18:24:16,335 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/B in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:16,336 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ee5dbcd683ee4fd7b13a38d0619becd8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/3b52874fec8040729e3bc062ec1521ae, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ea05e5509ce14a1e8b9a19104484fc28] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=36.8 K 2024-11-13T18:24:16,337 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee5dbcd683ee4fd7b13a38d0619becd8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1731522253265 2024-11-13T18:24:16,338 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b52874fec8040729e3bc062ec1521ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1731522253611 2024-11-13T18:24:16,338 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea05e5509ce14a1e8b9a19104484fc28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1731522254243 2024-11-13T18:24:16,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522316297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522316298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522316301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522316312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522316317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741914_1090 (size=17181) 2024-11-13T18:24:16,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/bb634d6ad1204100b938a2489f946ef6 2024-11-13T18:24:16,434 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:16,435 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/be3bd639a40246d684cf27381b10135f is 50, key is test_row_0/B:col10/1731522254261/Put/seqid=0 2024-11-13T18:24:16,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/28c99406e5984645b5664429a4280c1f is 50, key is test_row_0/B:col10/1731522256267/Put/seqid=0 2024-11-13T18:24:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522316466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522316467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522316467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741915_1091 (size=13153) 2024-11-13T18:24:16,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522316477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522316477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,528 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/be3bd639a40246d684cf27381b10135f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/be3bd639a40246d684cf27381b10135f 2024-11-13T18:24:16,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741916_1092 (size=12301) 2024-11-13T18:24:16,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/28c99406e5984645b5664429a4280c1f 2024-11-13T18:24:16,556 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/B of debb09c76feaf68052a52c8303bbb905 into be3bd639a40246d684cf27381b10135f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:16,556 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:16,556 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/B, priority=13, startTime=1731522256123; duration=0sec 2024-11-13T18:24:16,556 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:16,556 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:B 2024-11-13T18:24:16,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/93f6d5a2ed214bbb97b19561db3af17b is 50, key is test_row_0/C:col10/1731522256267/Put/seqid=0 2024-11-13T18:24:16,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741917_1093 (size=12301) 2024-11-13T18:24:16,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/93f6d5a2ed214bbb97b19561db3af17b 2024-11-13T18:24:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/bb634d6ad1204100b938a2489f946ef6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb634d6ad1204100b938a2489f946ef6 2024-11-13T18:24:16,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb634d6ad1204100b938a2489f946ef6, entries=250, sequenceid=678, filesize=16.8 K 2024-11-13T18:24:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:24:16,699 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522316697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,712 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/1eb484c6de9c47ff8dfd21f9f8e43d3d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/1eb484c6de9c47ff8dfd21f9f8e43d3d 2024-11-13T18:24:16,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/28c99406e5984645b5664429a4280c1f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/28c99406e5984645b5664429a4280c1f 2024-11-13T18:24:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-13T18:24:16,729 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:24:16,731 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:16,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:16,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522316717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522316719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522316719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/28c99406e5984645b5664429a4280c1f, entries=150, sequenceid=678, filesize=12.0 K 2024-11-13T18:24:16,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/93f6d5a2ed214bbb97b19561db3af17b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/93f6d5a2ed214bbb97b19561db3af17b 2024-11-13T18:24:16,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522316736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:16,787 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in debb09c76feaf68052a52c8303bbb905/C of debb09c76feaf68052a52c8303bbb905 into 1eb484c6de9c47ff8dfd21f9f8e43d3d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:16,787 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:16,787 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/C, priority=13, startTime=1731522256123; duration=0sec 2024-11-13T18:24:16,787 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:16,795 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:C 2024-11-13T18:24:16,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/93f6d5a2ed214bbb97b19561db3af17b, entries=150, sequenceid=678, filesize=12.0 K 2024-11-13T18:24:16,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~391.97 KB/401376, currentSize=40.25 KB/41220 for debb09c76feaf68052a52c8303bbb905 in 548ms, sequenceid=678, compaction requested=false 2024-11-13T18:24:16,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:24:16,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:24:16,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:16,912 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T18:24:16,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:16,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:16,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:16,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:16,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:16,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:16,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/b69c1903b57c436dbcc183b7eaa00e13 is 50, key is test_row_0/A:col10/1731522256310/Put/seqid=0 2024-11-13T18:24:17,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741918_1094 (size=12301) 2024-11-13T18:24:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:17,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. as already flushing 2024-11-13T18:24:17,047 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/b69c1903b57c436dbcc183b7eaa00e13 2024-11-13T18:24:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:24:17,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/50b5183074474f08934c63e0ab5f5b4e is 50, key is test_row_0/B:col10/1731522256310/Put/seqid=0 2024-11-13T18:24:17,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741919_1095 (size=12301) 2024-11-13T18:24:17,162 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/50b5183074474f08934c63e0ab5f5b4e 2024-11-13T18:24:17,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/aeb801794f1949ef873e09458ea80fa6 is 50, key is test_row_0/C:col10/1731522256310/Put/seqid=0 2024-11-13T18:24:17,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741920_1096 (size=12301) 2024-11-13T18:24:17,234 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/aeb801794f1949ef873e09458ea80fa6 2024-11-13T18:24:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/b69c1903b57c436dbcc183b7eaa00e13 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/b69c1903b57c436dbcc183b7eaa00e13 2024-11-13T18:24:17,310 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/b69c1903b57c436dbcc183b7eaa00e13, entries=150, sequenceid=690, filesize=12.0 K 2024-11-13T18:24:17,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/50b5183074474f08934c63e0ab5f5b4e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/50b5183074474f08934c63e0ab5f5b4e 2024-11-13T18:24:17,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522257111, max timestamp=1731522257312Num uniques 859; 2024-11-13T18:24:17,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:24:17,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522257111, max timestamp=1731522257312Num uniques 859; 2024-11-13T18:24:17,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:24:17,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522257111, max timestamp=1731522257312Num uniques 859; 2024-11-13T18:24:17,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:24:17,332 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/50b5183074474f08934c63e0ab5f5b4e, entries=150, sequenceid=690, filesize=12.0 K 2024-11-13T18:24:17,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/aeb801794f1949ef873e09458ea80fa6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/aeb801794f1949ef873e09458ea80fa6 2024-11-13T18:24:17,359 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/aeb801794f1949ef873e09458ea80fa6, entries=150, sequenceid=690, filesize=12.0 K 2024-11-13T18:24:17,363 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=59.20 KB/60617 for debb09c76feaf68052a52c8303bbb905 in 452ms, sequenceid=690, compaction requested=true 2024-11-13T18:24:17,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:17,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-13T18:24:17,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=65.91 KB heapSize=183.02 KB 2024-11-13T18:24:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-13T18:24:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:17,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T18:24:17,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 639 msec 2024-11-13T18:24:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:24:17,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f69b3403b86f49c3a9d104f639ef0e77 is 50, key is test_row_0/A:col10/1731522257362/Put/seqid=0 2024-11-13T18:24:17,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 668 msec 2024-11-13T18:24:17,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741921_1097 (size=26945) 2024-11-13T18:24:17,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522317475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522317485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522317485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522317494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522317513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522317616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522317626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522317628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,661 INFO [Thread-155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-155. 2024-11-13T18:24:17,662 DEBUG [Thread-155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:17,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522317635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522317656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,679 INFO [Thread-157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-157. 2024-11-13T18:24:17,679 DEBUG [Thread-157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:17,680 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:17,681 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:17,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:17,683 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:17,684 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:17,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:17,685 INFO [Thread-159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-159. 2024-11-13T18:24:17,686 DEBUG [Thread-159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:17,686 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:17,686 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:17,687 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:17,739 INFO [Thread-161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-161. 2024-11-13T18:24:17,740 DEBUG [Thread-161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:17,740 DEBUG [Thread-161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:17,740 DEBUG [Thread-161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:17,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:17,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522317850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522317850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.97 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f69b3403b86f49c3a9d104f639ef0e77 2024-11-13T18:24:17,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522317856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36988 deadline: 1731522317882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:24:17,897 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1731522317899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:17,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/2d759f2273f540fb8d21af134b551bf0 is 50, key is test_row_0/B:col10/1731522257362/Put/seqid=0 2024-11-13T18:24:17,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741922_1098 (size=12301) 2024-11-13T18:24:17,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.97 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/2d759f2273f540fb8d21af134b551bf0 2024-11-13T18:24:18,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/fb49f6a0a29d430f850586233317ca0e is 50, key is test_row_0/C:col10/1731522257362/Put/seqid=0 2024-11-13T18:24:18,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741923_1099 (size=12301) 2024-11-13T18:24:18,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.97 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/fb49f6a0a29d430f850586233317ca0e 2024-11-13T18:24:18,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/f69b3403b86f49c3a9d104f639ef0e77 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f69b3403b86f49c3a9d104f639ef0e77 2024-11-13T18:24:18,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f69b3403b86f49c3a9d104f639ef0e77, entries=450, sequenceid=717, filesize=26.3 K 2024-11-13T18:24:18,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/2d759f2273f540fb8d21af134b551bf0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/2d759f2273f540fb8d21af134b551bf0 2024-11-13T18:24:18,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/2d759f2273f540fb8d21af134b551bf0, entries=150, sequenceid=717, filesize=12.0 K 2024-11-13T18:24:18,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/fb49f6a0a29d430f850586233317ca0e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fb49f6a0a29d430f850586233317ca0e 2024-11-13T18:24:18,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36940 deadline: 1731522318166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:18,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1731522318171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:18,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36992 deadline: 1731522318183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:18,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fb49f6a0a29d430f850586233317ca0e, entries=150, sequenceid=717, filesize=12.0 K 2024-11-13T18:24:18,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~65.91 KB/67487, heapSize ~182.98 KB/187368, currentSize=114.05 KB/116790 for debb09c76feaf68052a52c8303bbb905 in 822ms, sequenceid=717, compaction requested=true 2024-11-13T18:24:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:18,189 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:24:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store debb09c76feaf68052a52c8303bbb905:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:24:18,193 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 69580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:24:18,193 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/A is initiating minor compaction (all files) 2024-11-13T18:24:18,193 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/A in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:18,194 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/3a48b29c5edf49b1a48b144a832128eb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb634d6ad1204100b938a2489f946ef6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/b69c1903b57c436dbcc183b7eaa00e13, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f69b3403b86f49c3a9d104f639ef0e77] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=67.9 K 2024-11-13T18:24:18,194 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:24:18,194 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a48b29c5edf49b1a48b144a832128eb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1731522254243 2024-11-13T18:24:18,197 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb634d6ad1204100b938a2489f946ef6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731522254962 2024-11-13T18:24:18,198 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b69c1903b57c436dbcc183b7eaa00e13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=690, earliestPutTs=1731522256294 2024-11-13T18:24:18,198 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting f69b3403b86f49c3a9d104f639ef0e77, keycount=450, bloomtype=ROW, size=26.3 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1731522257180 2024-11-13T18:24:18,199 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:24:18,199 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/B is initiating minor compaction (all files) 2024-11-13T18:24:18,199 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/B in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:18,199 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/be3bd639a40246d684cf27381b10135f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/28c99406e5984645b5664429a4280c1f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/50b5183074474f08934c63e0ab5f5b4e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/2d759f2273f540fb8d21af134b551bf0] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=48.9 K 2024-11-13T18:24:18,201 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting be3bd639a40246d684cf27381b10135f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1731522254243 2024-11-13T18:24:18,201 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 28c99406e5984645b5664429a4280c1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731522254987 2024-11-13T18:24:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:18,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T18:24:18,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:18,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:18,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:18,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:18,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:18,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:18,204 INFO [Thread-150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-150. 2024-11-13T18:24:18,205 DEBUG [Thread-150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:18,205 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,205 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:18,206 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 50b5183074474f08934c63e0ab5f5b4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=690, earliestPutTs=1731522256294 2024-11-13T18:24:18,207 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d759f2273f540fb8d21af134b551bf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1731522257348 2024-11-13T18:24:18,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/72e216b5c70d471e983013e151dab6f1 is 50, key is test_row_0/A:col10/1731522258199/Put/seqid=0 2024-11-13T18:24:18,227 INFO [Thread-148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-148. 2024-11-13T18:24:18,227 DEBUG [Thread-148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:18,227 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,228 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:18,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741924_1100 (size=12301) 2024-11-13T18:24:18,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/72e216b5c70d471e983013e151dab6f1 2024-11-13T18:24:18,329 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#B#compaction#88 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:18,330 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/7240b34dac84439abe8536ee72d1d426 is 50, key is test_row_0/B:col10/1731522257362/Put/seqid=0 2024-11-13T18:24:18,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741925_1101 (size=13289) 2024-11-13T18:24:18,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/42d9d9ca69824b0f94b6440d77db4b34 is 50, key is test_row_0/B:col10/1731522258199/Put/seqid=0 2024-11-13T18:24:18,447 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/7240b34dac84439abe8536ee72d1d426 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7240b34dac84439abe8536ee72d1d426 2024-11-13T18:24:18,490 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#A#compaction#89 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:18,491 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/dc4ec750dca1494ba77c727f94650518 is 50, key is test_row_0/A:col10/1731522257362/Put/seqid=0 2024-11-13T18:24:18,492 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in debb09c76feaf68052a52c8303bbb905/B of debb09c76feaf68052a52c8303bbb905 into 7240b34dac84439abe8536ee72d1d426(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:18,492 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:18,492 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/B, priority=12, startTime=1731522258189; duration=0sec 2024-11-13T18:24:18,493 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:18,493 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:B 2024-11-13T18:24:18,493 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:24:18,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741926_1102 (size=12301) 2024-11-13T18:24:18,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/42d9d9ca69824b0f94b6440d77db4b34 2024-11-13T18:24:18,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741927_1103 (size=13289) 2024-11-13T18:24:18,551 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:24:18,552 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): debb09c76feaf68052a52c8303bbb905/C is initiating minor compaction (all files) 2024-11-13T18:24:18,566 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of debb09c76feaf68052a52c8303bbb905/C in TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:18,571 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/1eb484c6de9c47ff8dfd21f9f8e43d3d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/93f6d5a2ed214bbb97b19561db3af17b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/aeb801794f1949ef873e09458ea80fa6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fb49f6a0a29d430f850586233317ca0e] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp, totalSize=48.9 K 2024-11-13T18:24:18,572 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb484c6de9c47ff8dfd21f9f8e43d3d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1731522254243 2024-11-13T18:24:18,575 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f6d5a2ed214bbb97b19561db3af17b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731522254987 2024-11-13T18:24:18,581 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/dc4ec750dca1494ba77c727f94650518 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/dc4ec750dca1494ba77c727f94650518 2024-11-13T18:24:18,581 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb801794f1949ef873e09458ea80fa6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=690, earliestPutTs=1731522256294 2024-11-13T18:24:18,583 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting fb49f6a0a29d430f850586233317ca0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1731522257348 2024-11-13T18:24:18,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/d5ff0e3d7601416b98c2dbc9e84a2d38 is 50, key is test_row_0/C:col10/1731522258199/Put/seqid=0 2024-11-13T18:24:18,607 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in debb09c76feaf68052a52c8303bbb905/A of debb09c76feaf68052a52c8303bbb905 into dc4ec750dca1494ba77c727f94650518(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:18,607 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:18,607 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/A, priority=12, startTime=1731522258189; duration=0sec 2024-11-13T18:24:18,607 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:18,607 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:A 2024-11-13T18:24:18,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741928_1104 (size=12301) 2024-11-13T18:24:18,681 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): debb09c76feaf68052a52c8303bbb905#C#compaction#92 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:18,682 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/09677ca243e140ca8eada2c1e7c308ed is 50, key is test_row_0/C:col10/1731522257362/Put/seqid=0 2024-11-13T18:24:18,695 INFO [Thread-146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-146. 2024-11-13T18:24:18,695 DEBUG [Thread-146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:18,695 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,695 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,696 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:18,696 INFO [Thread-152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-152. 2024-11-13T18:24:18,696 DEBUG [Thread-152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:18,696 DEBUG [Thread-152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,696 DEBUG [Thread-152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:18,700 INFO [Thread-144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-144. 2024-11-13T18:24:18,701 DEBUG [Thread-144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:18,701 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,701 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:18,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:24:18,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-13T18:24:18,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-13T18:24:18,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-13T18:24:18,705 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-13T18:24:18,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-13T18:24:18,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:24:18,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 606 2024-11-13T18:24:18,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 614 2024-11-13T18:24:18,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:24:18,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 319 2024-11-13T18:24:18,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 949 rows 2024-11-13T18:24:18,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 307 2024-11-13T18:24:18,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 920 rows 2024-11-13T18:24:18,706 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:24:18,706 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:24:18,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:18,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:18,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:24:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:18,757 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522258756"}]},"ts":"1731522258756"} 2024-11-13T18:24:18,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741929_1105 (size=13289) 2024-11-13T18:24:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:24:18,776 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:24:18,776 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:24:18,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:24:18,796 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/09677ca243e140ca8eada2c1e7c308ed as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/09677ca243e140ca8eada2c1e7c308ed 2024-11-13T18:24:18,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=debb09c76feaf68052a52c8303bbb905, UNASSIGN}] 2024-11-13T18:24:18,803 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=debb09c76feaf68052a52c8303bbb905, UNASSIGN 2024-11-13T18:24:18,806 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=debb09c76feaf68052a52c8303bbb905, regionState=CLOSING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:24:18,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=debb09c76feaf68052a52c8303bbb905, UNASSIGN because future has completed 2024-11-13T18:24:18,817 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in debb09c76feaf68052a52c8303bbb905/C of debb09c76feaf68052a52c8303bbb905 into 09677ca243e140ca8eada2c1e7c308ed(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:18,817 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:18,817 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., storeName=debb09c76feaf68052a52c8303bbb905/C, priority=12, startTime=1731522258189; duration=0sec 2024-11-13T18:24:18,817 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:18,817 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: debb09c76feaf68052a52c8303bbb905:C 2024-11-13T18:24:18,827 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:24:18,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:24:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:24:19,004 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:19,005 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:24:19,013 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing debb09c76feaf68052a52c8303bbb905, disabling compactions & flushes 2024-11-13T18:24:19,016 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:19,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/d5ff0e3d7601416b98c2dbc9e84a2d38 2024-11-13T18:24:19,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/72e216b5c70d471e983013e151dab6f1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/72e216b5c70d471e983013e151dab6f1 2024-11-13T18:24:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:24:19,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/72e216b5c70d471e983013e151dab6f1, entries=150, sequenceid=738, filesize=12.0 K 2024-11-13T18:24:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/42d9d9ca69824b0f94b6440d77db4b34 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/42d9d9ca69824b0f94b6440d77db4b34 2024-11-13T18:24:19,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/42d9d9ca69824b0f94b6440d77db4b34, entries=150, sequenceid=738, filesize=12.0 K 2024-11-13T18:24:19,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/d5ff0e3d7601416b98c2dbc9e84a2d38 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/d5ff0e3d7601416b98c2dbc9e84a2d38 2024-11-13T18:24:19,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/d5ff0e3d7601416b98c2dbc9e84a2d38, entries=150, sequenceid=738, filesize=12.0 K 2024-11-13T18:24:19,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=26.84 KB/27480 for debb09c76feaf68052a52c8303bbb905 in 938ms, sequenceid=738, compaction requested=false 2024-11-13T18:24:19,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for debb09c76feaf68052a52c8303bbb905: 2024-11-13T18:24:19,147 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:19,147 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:19,147 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. after waiting 0 ms 2024-11-13T18:24:19,147 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:19,147 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(2902): Flushing debb09c76feaf68052a52c8303bbb905 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T18:24:19,148 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=A 2024-11-13T18:24:19,148 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:19,148 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=B 2024-11-13T18:24:19,148 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:19,148 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK debb09c76feaf68052a52c8303bbb905, store=C 2024-11-13T18:24:19,148 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:19,173 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/514b11997e6b422d99c011aa78770f35 is 50, key is test_row_0/A:col10/1731522258219/Put/seqid=0 2024-11-13T18:24:19,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741930_1106 (size=12301) 2024-11-13T18:24:19,249 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/514b11997e6b422d99c011aa78770f35 2024-11-13T18:24:19,272 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/85a77e918c1c4065a189313730762250 is 50, key is test_row_0/B:col10/1731522258219/Put/seqid=0 2024-11-13T18:24:19,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741931_1107 (size=12301) 2024-11-13T18:24:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:24:19,707 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/85a77e918c1c4065a189313730762250 2024-11-13T18:24:19,765 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/9685b3a60a2c409f985a08dce5150361 is 50, key is test_row_0/C:col10/1731522258219/Put/seqid=0 2024-11-13T18:24:19,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741932_1108 (size=12301) 2024-11-13T18:24:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:24:20,254 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/9685b3a60a2c409f985a08dce5150361 2024-11-13T18:24:20,271 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/A/514b11997e6b422d99c011aa78770f35 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/514b11997e6b422d99c011aa78770f35 2024-11-13T18:24:20,308 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/514b11997e6b422d99c011aa78770f35, entries=150, sequenceid=748, filesize=12.0 K 2024-11-13T18:24:20,311 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/B/85a77e918c1c4065a189313730762250 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/85a77e918c1c4065a189313730762250 2024-11-13T18:24:20,332 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/85a77e918c1c4065a189313730762250, entries=150, sequenceid=748, filesize=12.0 K 2024-11-13T18:24:20,336 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/.tmp/C/9685b3a60a2c409f985a08dce5150361 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/9685b3a60a2c409f985a08dce5150361 2024-11-13T18:24:20,363 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/9685b3a60a2c409f985a08dce5150361, entries=150, sequenceid=748, filesize=12.0 K 2024-11-13T18:24:20,365 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for debb09c76feaf68052a52c8303bbb905 in 1218ms, sequenceid=748, compaction requested=true 2024-11-13T18:24:20,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/08b0ec93bbbe4b689bfb271cf98ba611, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/50b3c304a68a47f690fa6c333f9127f6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/c9b6225411854ae38e2bc95e83bf1f4b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/7b0e6b9da4f24fcd82dfeba35b56110e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2d041312db054064877ca15c9fc76b1a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/31c1904de95148368a2a483bcacabcff, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/698cbd2877a14ffc9e59b97b97bfcd5f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/6272981d9d62439f911a9a9fc0bb2588, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2af99cd069fb472493c16ef1f359f387, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/9ce285235dd54bcba5b7aaa5c9d4f64c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5ecf9e69d28c41f6af1d534e943c14de, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f5209c9775cc417ab7b8e8f6a2fd2fad, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/82a439a675a54e5d88f654e1f0d63983, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f0ea427823fa451c8004a873d0c7bdf3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f9d6f38df01b48a094b725b2c3beeca7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/1ae52dc5af1e42fc9788d6a5ffdf8903, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/0cb51084298b493db9235d494fc3bf16, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/44b2b1b08db7437282410688145d90c3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5a5b21aef69d4917b488c731dd443da4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5186fcea39974af3b77b37644ca7fec2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/e8db7227119948ffa3e7ca15c1489065, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/cead70067a8f4aa0b50ed86c519eb2bd, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb50b1280faa405a8d610e34c7b0a3ef, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/49618bb9c15d4db0a6549322e63d0cbe, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/3a48b29c5edf49b1a48b144a832128eb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/edca6cc3321a426f87f4da64b6859cf0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb634d6ad1204100b938a2489f946ef6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/b69c1903b57c436dbcc183b7eaa00e13, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f69b3403b86f49c3a9d104f639ef0e77] to archive 2024-11-13T18:24:20,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:24:20,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/08b0ec93bbbe4b689bfb271cf98ba611 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/08b0ec93bbbe4b689bfb271cf98ba611 2024-11-13T18:24:20,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/50b3c304a68a47f690fa6c333f9127f6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/50b3c304a68a47f690fa6c333f9127f6 2024-11-13T18:24:20,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/c9b6225411854ae38e2bc95e83bf1f4b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/c9b6225411854ae38e2bc95e83bf1f4b 2024-11-13T18:24:20,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/7b0e6b9da4f24fcd82dfeba35b56110e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/7b0e6b9da4f24fcd82dfeba35b56110e 2024-11-13T18:24:20,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2d041312db054064877ca15c9fc76b1a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2d041312db054064877ca15c9fc76b1a 2024-11-13T18:24:20,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/31c1904de95148368a2a483bcacabcff to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/31c1904de95148368a2a483bcacabcff 2024-11-13T18:24:20,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/698cbd2877a14ffc9e59b97b97bfcd5f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/698cbd2877a14ffc9e59b97b97bfcd5f 2024-11-13T18:24:20,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/6272981d9d62439f911a9a9fc0bb2588 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/6272981d9d62439f911a9a9fc0bb2588 2024-11-13T18:24:20,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2af99cd069fb472493c16ef1f359f387 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/2af99cd069fb472493c16ef1f359f387 2024-11-13T18:24:20,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/9ce285235dd54bcba5b7aaa5c9d4f64c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/9ce285235dd54bcba5b7aaa5c9d4f64c 2024-11-13T18:24:20,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5ecf9e69d28c41f6af1d534e943c14de to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5ecf9e69d28c41f6af1d534e943c14de 2024-11-13T18:24:20,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f5209c9775cc417ab7b8e8f6a2fd2fad to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f5209c9775cc417ab7b8e8f6a2fd2fad 2024-11-13T18:24:20,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/82a439a675a54e5d88f654e1f0d63983 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/82a439a675a54e5d88f654e1f0d63983 2024-11-13T18:24:20,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f0ea427823fa451c8004a873d0c7bdf3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f0ea427823fa451c8004a873d0c7bdf3 2024-11-13T18:24:20,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f9d6f38df01b48a094b725b2c3beeca7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f9d6f38df01b48a094b725b2c3beeca7 2024-11-13T18:24:20,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/1ae52dc5af1e42fc9788d6a5ffdf8903 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/1ae52dc5af1e42fc9788d6a5ffdf8903 2024-11-13T18:24:20,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/0cb51084298b493db9235d494fc3bf16 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/0cb51084298b493db9235d494fc3bf16 2024-11-13T18:24:20,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/44b2b1b08db7437282410688145d90c3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/44b2b1b08db7437282410688145d90c3 2024-11-13T18:24:20,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5a5b21aef69d4917b488c731dd443da4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5a5b21aef69d4917b488c731dd443da4 2024-11-13T18:24:20,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5186fcea39974af3b77b37644ca7fec2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/5186fcea39974af3b77b37644ca7fec2 2024-11-13T18:24:20,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/e8db7227119948ffa3e7ca15c1489065 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/e8db7227119948ffa3e7ca15c1489065 2024-11-13T18:24:20,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/cead70067a8f4aa0b50ed86c519eb2bd to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/cead70067a8f4aa0b50ed86c519eb2bd 2024-11-13T18:24:20,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb50b1280faa405a8d610e34c7b0a3ef to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb50b1280faa405a8d610e34c7b0a3ef 2024-11-13T18:24:20,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/49618bb9c15d4db0a6549322e63d0cbe to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/49618bb9c15d4db0a6549322e63d0cbe 2024-11-13T18:24:20,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/3a48b29c5edf49b1a48b144a832128eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/3a48b29c5edf49b1a48b144a832128eb 2024-11-13T18:24:20,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/edca6cc3321a426f87f4da64b6859cf0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/edca6cc3321a426f87f4da64b6859cf0 2024-11-13T18:24:20,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb634d6ad1204100b938a2489f946ef6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/bb634d6ad1204100b938a2489f946ef6 2024-11-13T18:24:20,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/b69c1903b57c436dbcc183b7eaa00e13 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/b69c1903b57c436dbcc183b7eaa00e13 2024-11-13T18:24:20,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f69b3403b86f49c3a9d104f639ef0e77 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/f69b3403b86f49c3a9d104f639ef0e77 2024-11-13T18:24:20,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/320813ab385b40f68bbb90d345294d59, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e05e9d4c91734ee4916ef8c299124d28, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/4c403657df9b4303bba3a50fe209633b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cf06300a43d94e38867db8d641caede6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cd4aac6c0170482fb0d0ee6003fc96ad, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/07851e8e62884969a2f7ce22f34af570, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/36aef2fd3e6d423fa7e7c7262fb55e05, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/a4e6556441de4d5a96f9616e07d1e3fa, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/be18beca70f24c169df66b0c05992041, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/efc16c7f436848f68e0e2044a5ca770b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7a8894d44fe74e33963db27ba11b0897, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/1df67042e76a40d988ca603fcc29f9ca, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/5947d3cf3c1249baaa409ca89fc38981, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7db7851e6bcc469889b88ac371f12fc6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/f780e71e3ee1421dbf39676a613f9721, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/9cfc6207f9aa4ac7bff2285ba3d4bd0d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/84a2259d56c546bd8bc58b496f4875ff, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/647d8f63200845e3af5447ece688f2a7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/fc8af82c7a8846bfb363038aaf6032ab, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/57247da1b89b4dd2ba72c7259946d57a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/212f488ff416473596c5873679c4653b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ee5dbcd683ee4fd7b13a38d0619becd8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e464575d7a8d45049a20f1a6ad56841f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/3b52874fec8040729e3bc062ec1521ae, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/be3bd639a40246d684cf27381b10135f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ea05e5509ce14a1e8b9a19104484fc28, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/28c99406e5984645b5664429a4280c1f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/50b5183074474f08934c63e0ab5f5b4e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/2d759f2273f540fb8d21af134b551bf0] to archive 2024-11-13T18:24:20,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:24:20,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/320813ab385b40f68bbb90d345294d59 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/320813ab385b40f68bbb90d345294d59 2024-11-13T18:24:20,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e05e9d4c91734ee4916ef8c299124d28 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e05e9d4c91734ee4916ef8c299124d28 2024-11-13T18:24:20,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/4c403657df9b4303bba3a50fe209633b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/4c403657df9b4303bba3a50fe209633b 2024-11-13T18:24:20,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cf06300a43d94e38867db8d641caede6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cf06300a43d94e38867db8d641caede6 2024-11-13T18:24:20,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cd4aac6c0170482fb0d0ee6003fc96ad to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/cd4aac6c0170482fb0d0ee6003fc96ad 2024-11-13T18:24:20,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/07851e8e62884969a2f7ce22f34af570 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/07851e8e62884969a2f7ce22f34af570 2024-11-13T18:24:20,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/36aef2fd3e6d423fa7e7c7262fb55e05 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/36aef2fd3e6d423fa7e7c7262fb55e05 2024-11-13T18:24:20,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/a4e6556441de4d5a96f9616e07d1e3fa to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/a4e6556441de4d5a96f9616e07d1e3fa 2024-11-13T18:24:20,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/be18beca70f24c169df66b0c05992041 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/be18beca70f24c169df66b0c05992041 2024-11-13T18:24:20,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/efc16c7f436848f68e0e2044a5ca770b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/efc16c7f436848f68e0e2044a5ca770b 2024-11-13T18:24:20,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7a8894d44fe74e33963db27ba11b0897 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7a8894d44fe74e33963db27ba11b0897 2024-11-13T18:24:20,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/1df67042e76a40d988ca603fcc29f9ca to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/1df67042e76a40d988ca603fcc29f9ca 2024-11-13T18:24:20,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/5947d3cf3c1249baaa409ca89fc38981 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/5947d3cf3c1249baaa409ca89fc38981 2024-11-13T18:24:20,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7db7851e6bcc469889b88ac371f12fc6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7db7851e6bcc469889b88ac371f12fc6 2024-11-13T18:24:20,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/f780e71e3ee1421dbf39676a613f9721 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/f780e71e3ee1421dbf39676a613f9721 2024-11-13T18:24:20,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/9cfc6207f9aa4ac7bff2285ba3d4bd0d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/9cfc6207f9aa4ac7bff2285ba3d4bd0d 2024-11-13T18:24:20,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/84a2259d56c546bd8bc58b496f4875ff to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/84a2259d56c546bd8bc58b496f4875ff 2024-11-13T18:24:20,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/647d8f63200845e3af5447ece688f2a7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/647d8f63200845e3af5447ece688f2a7 2024-11-13T18:24:20,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/fc8af82c7a8846bfb363038aaf6032ab to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/fc8af82c7a8846bfb363038aaf6032ab 2024-11-13T18:24:20,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/57247da1b89b4dd2ba72c7259946d57a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/57247da1b89b4dd2ba72c7259946d57a 2024-11-13T18:24:20,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/212f488ff416473596c5873679c4653b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/212f488ff416473596c5873679c4653b 2024-11-13T18:24:20,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ee5dbcd683ee4fd7b13a38d0619becd8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ee5dbcd683ee4fd7b13a38d0619becd8 2024-11-13T18:24:20,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e464575d7a8d45049a20f1a6ad56841f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/e464575d7a8d45049a20f1a6ad56841f 2024-11-13T18:24:20,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/3b52874fec8040729e3bc062ec1521ae to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/3b52874fec8040729e3bc062ec1521ae 2024-11-13T18:24:20,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/be3bd639a40246d684cf27381b10135f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/be3bd639a40246d684cf27381b10135f 2024-11-13T18:24:20,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ea05e5509ce14a1e8b9a19104484fc28 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/ea05e5509ce14a1e8b9a19104484fc28 2024-11-13T18:24:20,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/28c99406e5984645b5664429a4280c1f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/28c99406e5984645b5664429a4280c1f 2024-11-13T18:24:20,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/50b5183074474f08934c63e0ab5f5b4e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/50b5183074474f08934c63e0ab5f5b4e 2024-11-13T18:24:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:24:20,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/2d759f2273f540fb8d21af134b551bf0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/2d759f2273f540fb8d21af134b551bf0 2024-11-13T18:24:20,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/caed061ad3cc4ff3b45572e114c3821e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/7b0fd4c61c7741dd9bbb5c3904f22f81, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/82132888f8c641e78fa1e839bde93101, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/e43a46b79baa44d1bd4eca8eab89a6d9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/6d51249f5e8649a4a8513c64de37b854, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/257400b10abd4d8daeac0e9efc3caf73, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/70bd4188c9cf4657867a71a0c678daf6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/64d950561bd74a7384dc78a4b3895c97, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/d6fec423b61a4c29aaebc002a7d874d9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fe0a0ecd893e49848391d46d9ec485bf, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/3ab484d2d9f641a08bf9d2b4dd005879, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b92252f5d9ec4110b2de118aaf8527c4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/4e995d4ad4e04dcf95d15dbe1877052f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/62cb77324afa40c68c1bfd5808da2c5a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/a09a1e47d09c4d36bace2c1eff118844, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/905abe6c3c2e4754a7c006fdc4614def, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b17b881ab1d14acba926db993957219f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b0a19097ad4a4b1fa366cd61c9a60444, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b090e7f1a9544bd48c8221ef0dbb44a8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f5565aacc9b94f63aa8c694c6bca82d4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/9bf6a97debcb4e2baed2e5f02f027cd8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f4c7a2a103b048e29fd89c3f9e00c301, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/30e0d4c12b574420a46881e33fc7775e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/eaedd4fa38bd437f8093aeeae8c9669f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/1eb484c6de9c47ff8dfd21f9f8e43d3d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/2efccbd3e27549b99f6636c6463cef10, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/93f6d5a2ed214bbb97b19561db3af17b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/aeb801794f1949ef873e09458ea80fa6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fb49f6a0a29d430f850586233317ca0e] to archive 2024-11-13T18:24:20,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:24:20,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/caed061ad3cc4ff3b45572e114c3821e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/caed061ad3cc4ff3b45572e114c3821e 2024-11-13T18:24:20,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/7b0fd4c61c7741dd9bbb5c3904f22f81 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/7b0fd4c61c7741dd9bbb5c3904f22f81 2024-11-13T18:24:20,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/82132888f8c641e78fa1e839bde93101 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/82132888f8c641e78fa1e839bde93101 2024-11-13T18:24:20,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/e43a46b79baa44d1bd4eca8eab89a6d9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/e43a46b79baa44d1bd4eca8eab89a6d9 2024-11-13T18:24:20,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/6d51249f5e8649a4a8513c64de37b854 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/6d51249f5e8649a4a8513c64de37b854 2024-11-13T18:24:20,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/257400b10abd4d8daeac0e9efc3caf73 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/257400b10abd4d8daeac0e9efc3caf73 2024-11-13T18:24:20,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/70bd4188c9cf4657867a71a0c678daf6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/70bd4188c9cf4657867a71a0c678daf6 2024-11-13T18:24:21,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/64d950561bd74a7384dc78a4b3895c97 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/64d950561bd74a7384dc78a4b3895c97 2024-11-13T18:24:21,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/d6fec423b61a4c29aaebc002a7d874d9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/d6fec423b61a4c29aaebc002a7d874d9 2024-11-13T18:24:21,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fe0a0ecd893e49848391d46d9ec485bf to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fe0a0ecd893e49848391d46d9ec485bf 2024-11-13T18:24:21,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/3ab484d2d9f641a08bf9d2b4dd005879 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/3ab484d2d9f641a08bf9d2b4dd005879 2024-11-13T18:24:21,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b92252f5d9ec4110b2de118aaf8527c4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b92252f5d9ec4110b2de118aaf8527c4 2024-11-13T18:24:21,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/4e995d4ad4e04dcf95d15dbe1877052f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/4e995d4ad4e04dcf95d15dbe1877052f 2024-11-13T18:24:21,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/62cb77324afa40c68c1bfd5808da2c5a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/62cb77324afa40c68c1bfd5808da2c5a 2024-11-13T18:24:21,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/a09a1e47d09c4d36bace2c1eff118844 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/a09a1e47d09c4d36bace2c1eff118844 2024-11-13T18:24:21,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/905abe6c3c2e4754a7c006fdc4614def to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/905abe6c3c2e4754a7c006fdc4614def 2024-11-13T18:24:21,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b17b881ab1d14acba926db993957219f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b17b881ab1d14acba926db993957219f 2024-11-13T18:24:21,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b0a19097ad4a4b1fa366cd61c9a60444 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b0a19097ad4a4b1fa366cd61c9a60444 2024-11-13T18:24:21,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b090e7f1a9544bd48c8221ef0dbb44a8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/b090e7f1a9544bd48c8221ef0dbb44a8 2024-11-13T18:24:21,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f5565aacc9b94f63aa8c694c6bca82d4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f5565aacc9b94f63aa8c694c6bca82d4 2024-11-13T18:24:21,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/9bf6a97debcb4e2baed2e5f02f027cd8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/9bf6a97debcb4e2baed2e5f02f027cd8 2024-11-13T18:24:21,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f4c7a2a103b048e29fd89c3f9e00c301 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/f4c7a2a103b048e29fd89c3f9e00c301 2024-11-13T18:24:21,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/30e0d4c12b574420a46881e33fc7775e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/30e0d4c12b574420a46881e33fc7775e 2024-11-13T18:24:21,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/eaedd4fa38bd437f8093aeeae8c9669f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/eaedd4fa38bd437f8093aeeae8c9669f 2024-11-13T18:24:21,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/1eb484c6de9c47ff8dfd21f9f8e43d3d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/1eb484c6de9c47ff8dfd21f9f8e43d3d 2024-11-13T18:24:21,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/2efccbd3e27549b99f6636c6463cef10 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/2efccbd3e27549b99f6636c6463cef10 2024-11-13T18:24:21,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/93f6d5a2ed214bbb97b19561db3af17b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/93f6d5a2ed214bbb97b19561db3af17b 2024-11-13T18:24:21,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/aeb801794f1949ef873e09458ea80fa6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/aeb801794f1949ef873e09458ea80fa6 2024-11-13T18:24:21,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fb49f6a0a29d430f850586233317ca0e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/fb49f6a0a29d430f850586233317ca0e 2024-11-13T18:24:21,190 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/recovered.edits/751.seqid, newMaxSeqId=751, maxSeqId=1 2024-11-13T18:24:21,194 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. 2024-11-13T18:24:21,195 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for debb09c76feaf68052a52c8303bbb905: Waiting for close lock at 1731522259010Running coprocessor pre-close hooks at 1731522259013 (+3 ms)Disabling compacts and flushes for region at 1731522259013Disabling writes for close at 1731522259147 (+134 ms)Obtaining lock to block concurrent updates at 1731522259147Preparing flush snapshotting stores in debb09c76feaf68052a52c8303bbb905 at 1731522259147Finished memstore snapshotting TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731522259148 (+1 ms)Flushing stores of TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905. at 1731522259158 (+10 ms)Flushing debb09c76feaf68052a52c8303bbb905/A: creating writer at 1731522259158Flushing debb09c76feaf68052a52c8303bbb905/A: appending metadata at 1731522259172 (+14 ms)Flushing debb09c76feaf68052a52c8303bbb905/A: closing flushed file at 1731522259173 (+1 ms)Flushing debb09c76feaf68052a52c8303bbb905/B: creating writer at 1731522259261 (+88 ms)Flushing debb09c76feaf68052a52c8303bbb905/B: appending metadata at 1731522259272 (+11 ms)Flushing debb09c76feaf68052a52c8303bbb905/B: closing flushed file at 1731522259272Flushing debb09c76feaf68052a52c8303bbb905/C: creating writer at 1731522259748 (+476 ms)Flushing debb09c76feaf68052a52c8303bbb905/C: appending metadata at 1731522259764 (+16 ms)Flushing debb09c76feaf68052a52c8303bbb905/C: closing flushed file at 1731522259764Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26b1502e: reopening flushed file at 1731522260263 (+499 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f53c849: reopening flushed file at 1731522260308 (+45 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66d01b93: reopening flushed file at 1731522260332 (+24 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for debb09c76feaf68052a52c8303bbb905 in 1218ms, sequenceid=748, compaction requested=true at 1731522260365 (+33 ms)Writing region close event to WAL at 1731522261178 (+813 ms)Running coprocessor post-close hooks at 1731522261191 (+13 ms)Closed at 1731522261194 (+3 ms) 2024-11-13T18:24:21,200 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:21,203 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=debb09c76feaf68052a52c8303bbb905, regionState=CLOSED 2024-11-13T18:24:21,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:24:21,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-13T18:24:21,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure debb09c76feaf68052a52c8303bbb905, server=62ae331128c9,39615,1731522225114 in 2.4100 sec 2024-11-13T18:24:21,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-13T18:24:21,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=debb09c76feaf68052a52c8303bbb905, UNASSIGN in 2.4510 sec 2024-11-13T18:24:21,265 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522261265"}]},"ts":"1731522261265"} 2024-11-13T18:24:21,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T18:24:21,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4740 sec 2024-11-13T18:24:21,268 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:24:21,268 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:24:21,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5340 sec 2024-11-13T18:24:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:24:22,931 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:22,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:24:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:22,963 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:22,966 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:24:22,985 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:23,015 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/recovered.edits] 2024-11-13T18:24:23,022 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/514b11997e6b422d99c011aa78770f35 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/514b11997e6b422d99c011aa78770f35 2024-11-13T18:24:23,031 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/72e216b5c70d471e983013e151dab6f1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/72e216b5c70d471e983013e151dab6f1 2024-11-13T18:24:23,038 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/dc4ec750dca1494ba77c727f94650518 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/A/dc4ec750dca1494ba77c727f94650518 2024-11-13T18:24:23,045 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/42d9d9ca69824b0f94b6440d77db4b34 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/42d9d9ca69824b0f94b6440d77db4b34 2024-11-13T18:24:23,048 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7240b34dac84439abe8536ee72d1d426 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/7240b34dac84439abe8536ee72d1d426 2024-11-13T18:24:23,052 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/85a77e918c1c4065a189313730762250 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/B/85a77e918c1c4065a189313730762250 2024-11-13T18:24:23,065 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/09677ca243e140ca8eada2c1e7c308ed to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/09677ca243e140ca8eada2c1e7c308ed 2024-11-13T18:24:23,070 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/9685b3a60a2c409f985a08dce5150361 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/9685b3a60a2c409f985a08dce5150361 2024-11-13T18:24:23,081 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/d5ff0e3d7601416b98c2dbc9e84a2d38 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/C/d5ff0e3d7601416b98c2dbc9e84a2d38 2024-11-13T18:24:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:24:23,098 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/recovered.edits/751.seqid to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905/recovered.edits/751.seqid 2024-11-13T18:24:23,101 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/debb09c76feaf68052a52c8303bbb905 2024-11-13T18:24:23,102 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:24:23,113 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:23,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-13T18:24:23,128 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:24:23,235 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:24:23,237 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:23,237 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:24:23,244 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731522263237"}]},"ts":"9223372036854775807"} 2024-11-13T18:24:23,252 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:24:23,252 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => debb09c76feaf68052a52c8303bbb905, NAME => 'TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:24:23,253 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:24:23,253 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731522263253"}]},"ts":"9223372036854775807"} 2024-11-13T18:24:23,262 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:24:23,268 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:23,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 329 msec 2024-11-13T18:24:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:24:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:24:23,301 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:23,336 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 209) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;62ae331128c9:39615-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-536054504_22 at /127.0.0.1:59306 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_633754790_22 at /127.0.0.1:40190 [Waiting for operation #267] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/62ae331128c9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_633754790_22 at /127.0.0.1:40146 [Waiting for operation #296] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=2178 (was 2002) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=857 (was 1893) 2024-11-13T18:24:23,371 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=2178, ProcessCount=11, AvailableMemoryMB=855 2024-11-13T18:24:23,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:24:23,374 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:24:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:23,378 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:24:23,378 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:23,380 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-11-13T18:24:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:24:23,382 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:24:23,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741933_1109 (size=1061) 2024-11-13T18:24:23,463 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b 2024-11-13T18:24:23,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:24:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:24:23,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741934_1110 (size=53) 2024-11-13T18:24:23,526 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:24:23,526 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a3e5c2fc3660c99b882b8a9ec24a558f, disabling compactions & flushes 2024-11-13T18:24:23,526 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:23,526 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:23,526 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. after waiting 0 ms 2024-11-13T18:24:23,526 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:23,527 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:23,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a3e5c2fc3660c99b882b8a9ec24a558f: Waiting for close lock at 1731522263526Disabling compacts and flushes for region at 1731522263526Disabling writes for close at 1731522263526Writing region close event to WAL at 1731522263526Closed at 1731522263526 2024-11-13T18:24:23,528 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:24:23,529 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731522263529"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731522263529"}]},"ts":"1731522263529"} 2024-11-13T18:24:23,533 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:24:23,534 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:24:23,535 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522263534"}]},"ts":"1731522263534"} 2024-11-13T18:24:23,540 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:24:23,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, ASSIGN}] 2024-11-13T18:24:23,545 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, ASSIGN 2024-11-13T18:24:23,546 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, ASSIGN; state=OFFLINE, location=62ae331128c9,39615,1731522225114; forceNewPlan=false, retain=false 2024-11-13T18:24:23,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=a3e5c2fc3660c99b882b8a9ec24a558f, regionState=OPENING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:24:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:24:23,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, ASSIGN because future has completed 2024-11-13T18:24:23,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:24:23,895 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:23,895 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:24:23,899 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,902 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:24:23,903 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,903 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,913 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,920 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:24:23,921 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e5c2fc3660c99b882b8a9ec24a558f columnFamilyName A 2024-11-13T18:24:23,921 DEBUG [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:23,922 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(327): Store=a3e5c2fc3660c99b882b8a9ec24a558f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:24:23,922 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,927 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:24:23,928 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e5c2fc3660c99b882b8a9ec24a558f columnFamilyName B 2024-11-13T18:24:23,928 DEBUG [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:23,931 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(327): Store=a3e5c2fc3660c99b882b8a9ec24a558f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:24:23,931 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,935 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:24:23,935 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e5c2fc3660c99b882b8a9ec24a558f columnFamilyName C 2024-11-13T18:24:23,935 DEBUG [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:23,947 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(327): Store=a3e5c2fc3660c99b882b8a9ec24a558f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:24:23,947 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:23,947 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,948 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,949 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,951 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,951 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,952 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:24:23,966 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,992 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:24:23,993 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened a3e5c2fc3660c99b882b8a9ec24a558f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71116599, jitterRate=0.05971990525722504}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:24:23,993 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:23,999 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for a3e5c2fc3660c99b882b8a9ec24a558f: Running coprocessor pre-open hook at 1731522263903Writing region info on filesystem at 1731522263903Initializing all the Stores at 1731522263912 (+9 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522263913 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522263913Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522263913Cleaning up temporary data from old regions at 1731522263951 (+38 ms)Running coprocessor post-open hooks at 1731522263993 (+42 ms)Region opened successfully at 1731522263999 (+6 ms) 2024-11-13T18:24:24,000 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., pid=26, masterSystemTime=1731522263875 2024-11-13T18:24:24,004 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:24,004 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:24,007 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=a3e5c2fc3660c99b882b8a9ec24a558f, regionState=OPEN, openSeqNum=2, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:24:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:24:24,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:24:24,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T18:24:24,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 in 302 msec 2024-11-13T18:24:24,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-13T18:24:24,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, ASSIGN in 488 msec 2024-11-13T18:24:24,035 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:24:24,035 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522264035"}]},"ts":"1731522264035"} 2024-11-13T18:24:24,042 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:24:24,044 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:24:24,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 669 msec 2024-11-13T18:24:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:24:24,527 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:24,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bda5726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:24,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:24,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:24,543 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3033f381, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:24,547 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4653ce58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:24,549 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:24,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:24,556 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:24:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 62ae331128c9,45483,1731522221514 2024-11-13T18:24:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@73c73a54 2024-11-13T18:24:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:24:24,564 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:24:24,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:24:24,575 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:24:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:24,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741935_1111 (size=1097) 2024-11-13T18:24:24,718 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T18:24:24,718 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T18:24:24,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:24:24,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, REOPEN/MOVE}] 2024-11-13T18:24:24,753 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, REOPEN/MOVE 2024-11-13T18:24:24,757 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a3e5c2fc3660c99b882b8a9ec24a558f, regionState=CLOSING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:24:24,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, REOPEN/MOVE because future has completed 2024-11-13T18:24:24,776 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45483 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=62ae331128c9,39615,1731522225114, table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:24:24,780 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:24:24,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:24:24,937 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:24,938 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:24:24,938 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing a3e5c2fc3660c99b882b8a9ec24a558f, disabling compactions & flushes 2024-11-13T18:24:24,938 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:24,938 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:24,938 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. after waiting 0 ms 2024-11-13T18:24:24,938 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:24,956 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:24:24,957 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:24,958 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for a3e5c2fc3660c99b882b8a9ec24a558f: Waiting for close lock at 1731522264938Running coprocessor pre-close hooks at 1731522264938Disabling compacts and flushes for region at 1731522264938Disabling writes for close at 1731522264938Writing region close event to WAL at 1731522264943 (+5 ms)Running coprocessor post-close hooks at 1731522264957 (+14 ms)Closed at 1731522264957 2024-11-13T18:24:24,958 WARN [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: a3e5c2fc3660c99b882b8a9ec24a558f to self. 2024-11-13T18:24:24,961 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:24,967 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a3e5c2fc3660c99b882b8a9ec24a558f, regionState=CLOSED 2024-11-13T18:24:24,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:24:24,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T18:24:24,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 in 194 msec 2024-11-13T18:24:24,979 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, REOPEN/MOVE; state=CLOSED, location=62ae331128c9,39615,1731522225114; forceNewPlan=false, retain=true 2024-11-13T18:24:25,130 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a3e5c2fc3660c99b882b8a9ec24a558f, regionState=OPENING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:24:25,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, REOPEN/MOVE because future has completed 2024-11-13T18:24:25,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:24:25,316 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:25,316 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:24:25,317 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,317 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:24:25,317 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,317 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,339 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,343 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:24:25,349 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e5c2fc3660c99b882b8a9ec24a558f columnFamilyName A 2024-11-13T18:24:25,372 DEBUG [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:25,373 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(327): Store=a3e5c2fc3660c99b882b8a9ec24a558f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:24:25,373 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,374 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:24:25,375 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e5c2fc3660c99b882b8a9ec24a558f columnFamilyName B 2024-11-13T18:24:25,375 DEBUG [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:25,375 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(327): Store=a3e5c2fc3660c99b882b8a9ec24a558f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:24:25,376 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,376 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:24:25,377 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3e5c2fc3660c99b882b8a9ec24a558f columnFamilyName C 2024-11-13T18:24:25,377 DEBUG [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:25,378 INFO [StoreOpener-a3e5c2fc3660c99b882b8a9ec24a558f-1 {}] regionserver.HStore(327): Store=a3e5c2fc3660c99b882b8a9ec24a558f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:24:25,378 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:25,378 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,379 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,384 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,391 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,391 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,397 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:24:25,400 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,402 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened a3e5c2fc3660c99b882b8a9ec24a558f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70921453, jitterRate=0.05681200325489044}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:24:25,402 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,405 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for a3e5c2fc3660c99b882b8a9ec24a558f: Running coprocessor pre-open hook at 1731522265317Writing region info on filesystem at 1731522265317Initializing all the Stores at 1731522265319 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522265319Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522265334 (+15 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522265335 (+1 ms)Cleaning up temporary data from old regions at 1731522265391 (+56 ms)Running coprocessor post-open hooks at 1731522265402 (+11 ms)Region opened successfully at 1731522265405 (+3 ms) 2024-11-13T18:24:25,407 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., pid=31, masterSystemTime=1731522265297 2024-11-13T18:24:25,410 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:25,411 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:25,411 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a3e5c2fc3660c99b882b8a9ec24a558f, regionState=OPEN, openSeqNum=5, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:24:25,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:24:25,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-13T18:24:25,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 in 281 msec 2024-11-13T18:24:25,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-13T18:24:25,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, REOPEN/MOVE in 675 msec 2024-11-13T18:24:25,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 860 msec 2024-11-13T18:24:25,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-13T18:24:25,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 703 msec 2024-11-13T18:24:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T18:24:25,451 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:25,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b697fdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:25,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:25,467 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71874a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:25,481 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26ee7c08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d441be3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:25,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:25,502 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87cd139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:25,525 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:25,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6fbc97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e51c7cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:25,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:25,531 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66250196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:25,535 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56dd2f11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f60a924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:25,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:25,559 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74326831, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:25,563 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c42f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f86a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:25,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:25,583 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eec3cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:25,591 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666da7c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4533212, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:25,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:25,602 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503bc05a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:25,611 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:25,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a093fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776350f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:25,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:25,631 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4308d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:25,635 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:25,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26146576, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22268491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:25,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:25,641 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ef9cd41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:25,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:25,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:25,645 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23786a82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239745cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:25,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:25,660 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747a78cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:25,663 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df3039, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:25,678 DEBUG [Thread-583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:25,681 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:25,682 DEBUG [Thread-583 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:25,682 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:25,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:25,682 DEBUG [Thread-583 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:25,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:25,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:25,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:25,684 DEBUG [Thread-585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:25,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:25,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:25,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:25,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:25,699 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:25,700 DEBUG [Thread-581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:25,708 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:25,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:25,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:25,728 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:25,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:25,742 DEBUG [Thread-579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:25,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:25,745 DEBUG [Thread-579 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:25,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:25,769 DEBUG [Thread-596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:24:25,771 DEBUG [Thread-596 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:25,771 DEBUG [Thread-596 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:25,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:24:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:24:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:24:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:24:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:24:25,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:24:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-13T18:24:25,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:24:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:24:25,790 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:24:25,795 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:25,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:24:25,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:25,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:25,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:25,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:25,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:25,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:24:25,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139dec8de5f8ff45a0a860ed8b7af4a0f0_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522265794/Put/seqid=0 2024-11-13T18:24:25,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731522265838, max timestamp=1731522265934Num uniques 859; 2024-11-13T18:24:25,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731522265838, max timestamp=1731522265934Num uniques 859; 2024-11-13T18:24:25,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:24:25,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:24:25,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731522265838, max timestamp=1731522265934Num uniques 859; 2024-11-13T18:24:25,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:24:25,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741936_1112 (size=14594) 2024-11-13T18:24:25,966 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:25,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:24:25,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:25,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:25,975 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139dec8de5f8ff45a0a860ed8b7af4a0f0_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139dec8de5f8ff45a0a860ed8b7af4a0f0_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:25,979 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/b184e7ff85a8440697d74c2045b24857, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:25,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/b184e7ff85a8440697d74c2045b24857 is 175, key is test_row_0/A:col10/1731522265794/Put/seqid=0 2024-11-13T18:24:26,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741937_1113 (size=39549) 2024-11-13T18:24:26,028 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/b184e7ff85a8440697d74c2045b24857 2024-11-13T18:24:26,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522326028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/469bc2952f9c4465a0f8925c52a06704 is 50, key is test_row_0/B:col10/1731522265794/Put/seqid=0 2024-11-13T18:24:26,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:24:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522326061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522326066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522326068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522326070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741938_1114 (size=12001) 2024-11-13T18:24:26,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:24:26,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:26,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522326180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522326216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522326217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522326227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522326217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:24:26,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:26,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:24:26,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522326407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:24:26,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:26,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522326456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522326456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522326466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522326466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/469bc2952f9c4465a0f8925c52a06704 2024-11-13T18:24:26,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:24:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/62a263eef6f941c0b2a477de1991ea9b is 50, key is test_row_0/C:col10/1731522265794/Put/seqid=0 2024-11-13T18:24:26,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741939_1115 (size=12001) 2024-11-13T18:24:26,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522326748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:24:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,786 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:24:26,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522326776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522326787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522326796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522326796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:24:26,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:24:26,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:26,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:26,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:27,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/62a263eef6f941c0b2a477de1991ea9b 2024-11-13T18:24:27,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/b184e7ff85a8440697d74c2045b24857 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/b184e7ff85a8440697d74c2045b24857 2024-11-13T18:24:27,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/b184e7ff85a8440697d74c2045b24857, entries=200, sequenceid=14, filesize=38.6 K 2024-11-13T18:24:27,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/469bc2952f9c4465a0f8925c52a06704 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/469bc2952f9c4465a0f8925c52a06704 2024-11-13T18:24:27,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:24:27,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:27,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:27,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/469bc2952f9c4465a0f8925c52a06704, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T18:24:27,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/62a263eef6f941c0b2a477de1991ea9b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62a263eef6f941c0b2a477de1991ea9b 2024-11-13T18:24:27,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62a263eef6f941c0b2a477de1991ea9b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T18:24:27,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a3e5c2fc3660c99b882b8a9ec24a558f in 1322ms, sequenceid=14, compaction requested=false 2024-11-13T18:24:27,134 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T18:24:27,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:27,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:24:27,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:27,254 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-13T18:24:27,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:27,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:27,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:27,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:27,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dce5a2074f6a4149a7f105381ff368a8_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522266014/Put/seqid=0 2024-11-13T18:24:27,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:27,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741940_1116 (size=12154) 2024-11-13T18:24:27,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:27,393 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dce5a2074f6a4149a7f105381ff368a8_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dce5a2074f6a4149a7f105381ff368a8_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:27,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522327375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522327379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522327381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522327382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/c67d1c5820d3472d84bf957e4a763557, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/c67d1c5820d3472d84bf957e4a763557 is 175, key is test_row_0/A:col10/1731522266014/Put/seqid=0 2024-11-13T18:24:27,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522327383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741941_1117 (size=30955) 2024-11-13T18:24:27,488 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/c67d1c5820d3472d84bf957e4a763557 2024-11-13T18:24:27,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522327516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522327517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522327517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522327517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/1eb46c700e8d46a5a82c6e782d63f181 is 50, key is test_row_0/B:col10/1731522266014/Put/seqid=0 2024-11-13T18:24:27,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522327527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741942_1118 (size=12001) 2024-11-13T18:24:27,617 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/1eb46c700e8d46a5a82c6e782d63f181 2024-11-13T18:24:27,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/e2a6f263ed3b4403b50b0d57fdd46467 is 50, key is test_row_0/C:col10/1731522266014/Put/seqid=0 2024-11-13T18:24:27,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741943_1119 (size=12001) 2024-11-13T18:24:27,709 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/e2a6f263ed3b4403b50b0d57fdd46467 2024-11-13T18:24:27,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/c67d1c5820d3472d84bf957e4a763557 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c67d1c5820d3472d84bf957e4a763557 2024-11-13T18:24:27,725 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c67d1c5820d3472d84bf957e4a763557, entries=150, sequenceid=49, filesize=30.2 K 2024-11-13T18:24:27,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/1eb46c700e8d46a5a82c6e782d63f181 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1eb46c700e8d46a5a82c6e782d63f181 2024-11-13T18:24:27,738 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1eb46c700e8d46a5a82c6e782d63f181, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T18:24:27,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/e2a6f263ed3b4403b50b0d57fdd46467 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e2a6f263ed3b4403b50b0d57fdd46467 2024-11-13T18:24:27,752 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e2a6f263ed3b4403b50b0d57fdd46467, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T18:24:27,753 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for a3e5c2fc3660c99b882b8a9ec24a558f in 499ms, sequenceid=49, compaction requested=false 2024-11-13T18:24:27,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:27,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:27,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-13T18:24:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-13T18:24:27,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-13T18:24:27,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9610 sec 2024-11-13T18:24:27,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 2.0530 sec 2024-11-13T18:24:27,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:24:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:27,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:27,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:27,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:27,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:27,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:27,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:27,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137935be3e047c4cc384922405e5dd3722_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522267378/Put/seqid=0 2024-11-13T18:24:27,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741944_1120 (size=12154) 2024-11-13T18:24:27,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:27,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522327852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522327856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,880 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137935be3e047c4cc384922405e5dd3722_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137935be3e047c4cc384922405e5dd3722_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/c04bb0259ccf41d7a00130182e505a5b, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:27,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/c04bb0259ccf41d7a00130182e505a5b is 175, key is test_row_0/A:col10/1731522267378/Put/seqid=0 2024-11-13T18:24:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522327864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522327869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522327870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741945_1121 (size=30955) 2024-11-13T18:24:27,938 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/c04bb0259ccf41d7a00130182e505a5b 2024-11-13T18:24:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:24:27,948 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:27,955 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-13T18:24:27,962 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:27,964 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:27,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:24:27,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/ec99361e07f9471681e85d9bf4881e15 is 50, key is test_row_0/B:col10/1731522267378/Put/seqid=0 2024-11-13T18:24:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522327986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522327995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741946_1122 (size=12001) 2024-11-13T18:24:28,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/ec99361e07f9471681e85d9bf4881e15 2024-11-13T18:24:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:24:28,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/0bc4df3da9f347e786d9bb7dcdf6793b is 50, key is test_row_0/C:col10/1731522267378/Put/seqid=0 2024-11-13T18:24:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522328037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522328036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522328059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741947_1123 (size=12001) 2024-11-13T18:24:28,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:24:28,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:28,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:28,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:28,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:28,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:28,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/0bc4df3da9f347e786d9bb7dcdf6793b 2024-11-13T18:24:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:28,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/c04bb0259ccf41d7a00130182e505a5b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c04bb0259ccf41d7a00130182e505a5b 2024-11-13T18:24:28,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c04bb0259ccf41d7a00130182e505a5b, entries=150, sequenceid=62, filesize=30.2 K 2024-11-13T18:24:28,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/ec99361e07f9471681e85d9bf4881e15 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/ec99361e07f9471681e85d9bf4881e15 2024-11-13T18:24:28,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/ec99361e07f9471681e85d9bf4881e15, entries=150, sequenceid=62, filesize=11.7 K 2024-11-13T18:24:28,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/0bc4df3da9f347e786d9bb7dcdf6793b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0bc4df3da9f347e786d9bb7dcdf6793b 2024-11-13T18:24:28,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522328227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522328227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0bc4df3da9f347e786d9bb7dcdf6793b, entries=150, sequenceid=62, filesize=11.7 K 2024-11-13T18:24:28,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a3e5c2fc3660c99b882b8a9ec24a558f in 480ms, sequenceid=62, compaction requested=true 2024-11-13T18:24:28,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:28,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:28,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:28,250 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:28,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:28,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:28,251 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:28,252 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:28,252 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/A is initiating minor compaction (all files) 2024-11-13T18:24:28,253 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/A in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:28,253 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/b184e7ff85a8440697d74c2045b24857, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c67d1c5820d3472d84bf957e4a763557, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c04bb0259ccf41d7a00130182e505a5b] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=99.1 K 2024-11-13T18:24:28,253 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:28,253 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/b184e7ff85a8440697d74c2045b24857, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c67d1c5820d3472d84bf957e4a763557, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c04bb0259ccf41d7a00130182e505a5b] 2024-11-13T18:24:28,254 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:28,254 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/B is initiating minor compaction (all files) 2024-11-13T18:24:28,254 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/B in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:28,255 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/469bc2952f9c4465a0f8925c52a06704, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1eb46c700e8d46a5a82c6e782d63f181, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/ec99361e07f9471681e85d9bf4881e15] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=35.2 K 2024-11-13T18:24:28,255 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting b184e7ff85a8440697d74c2045b24857, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731522265789 2024-11-13T18:24:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,256 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 469bc2952f9c4465a0f8925c52a06704, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731522265789 2024-11-13T18:24:28,257 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c67d1c5820d3472d84bf957e4a763557, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731522266014 2024-11-13T18:24:28,257 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eb46c700e8d46a5a82c6e782d63f181, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731522266014 2024-11-13T18:24:28,258 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c04bb0259ccf41d7a00130182e505a5b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522267378 2024-11-13T18:24:28,259 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec99361e07f9471681e85d9bf4881e15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522267378 2024-11-13T18:24:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:24:28,292 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:28,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:24:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:28,298 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:24:28,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:28,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:28,326 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#B#compaction#106 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:28,327 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/6384f42ea378480b98e87d28fa6e3e3d is 50, key is test_row_0/B:col10/1731522267378/Put/seqid=0 2024-11-13T18:24:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:28,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:28,336 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138c69ad7d91d1455eae670b195ab4e0d8_a3e5c2fc3660c99b882b8a9ec24a558f store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:28,354 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138c69ad7d91d1455eae670b195ab4e0d8_a3e5c2fc3660c99b882b8a9ec24a558f, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:28,354 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138c69ad7d91d1455eae670b195ab4e0d8_a3e5c2fc3660c99b882b8a9ec24a558f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:28,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d911d8ef59f04d22a61874855b39ae1b_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522267851/Put/seqid=0 2024-11-13T18:24:28,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522328360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522328380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522328389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741948_1124 (size=12104) 2024-11-13T18:24:28,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741949_1125 (size=4469) 2024-11-13T18:24:28,480 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#A#compaction#105 average throughput is 0.13 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:28,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741950_1126 (size=12154) 2024-11-13T18:24:28,481 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/86098d2ca20846a4a5725d22aa222a4e is 175, key is test_row_0/A:col10/1731522267378/Put/seqid=0 2024-11-13T18:24:28,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:28,499 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d911d8ef59f04d22a61874855b39ae1b_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d911d8ef59f04d22a61874855b39ae1b_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/d829ee2830584ee0b967de22452fedc4, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/d829ee2830584ee0b967de22452fedc4 is 175, key is test_row_0/A:col10/1731522267851/Put/seqid=0 2024-11-13T18:24:28,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522328496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522328506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522328518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741952_1128 (size=30955) 2024-11-13T18:24:28,563 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/d829ee2830584ee0b967de22452fedc4 2024-11-13T18:24:28,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741951_1127 (size=31058) 2024-11-13T18:24:28,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/f898ac289ea84bceb547496d90a28929 is 50, key is test_row_0/B:col10/1731522267851/Put/seqid=0 2024-11-13T18:24:28,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522328568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:24:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522328551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,606 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/86098d2ca20846a4a5725d22aa222a4e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/86098d2ca20846a4a5725d22aa222a4e 2024-11-13T18:24:28,632 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/A of a3e5c2fc3660c99b882b8a9ec24a558f into 86098d2ca20846a4a5725d22aa222a4e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:28,632 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:28,632 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/A, priority=13, startTime=1731522268250; duration=0sec 2024-11-13T18:24:28,632 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:28,632 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:A 2024-11-13T18:24:28,632 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:28,637 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:28,637 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/C is initiating minor compaction (all files) 2024-11-13T18:24:28,637 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/C in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:28,637 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62a263eef6f941c0b2a477de1991ea9b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e2a6f263ed3b4403b50b0d57fdd46467, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0bc4df3da9f347e786d9bb7dcdf6793b] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=35.2 K 2024-11-13T18:24:28,638 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 62a263eef6f941c0b2a477de1991ea9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731522265789 2024-11-13T18:24:28,638 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e2a6f263ed3b4403b50b0d57fdd46467, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731522266014 2024-11-13T18:24:28,639 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bc4df3da9f347e786d9bb7dcdf6793b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522267378 2024-11-13T18:24:28,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741953_1129 (size=12001) 2024-11-13T18:24:28,653 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/f898ac289ea84bceb547496d90a28929 2024-11-13T18:24:28,680 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#C#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:28,680 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/955e98f1345b4cb688ac9677c9a76f82 is 50, key is test_row_0/C:col10/1731522267378/Put/seqid=0 2024-11-13T18:24:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/4b18806df3f84f11854a7eddfb0ed1ab is 50, key is test_row_0/C:col10/1731522267851/Put/seqid=0 2024-11-13T18:24:28,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741954_1130 (size=12104) 2024-11-13T18:24:28,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522328736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522328741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522328741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:28,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:28,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:28,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741955_1131 (size=12001) 2024-11-13T18:24:28,778 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/4b18806df3f84f11854a7eddfb0ed1ab 2024-11-13T18:24:28,799 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/955e98f1345b4cb688ac9677c9a76f82 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/955e98f1345b4cb688ac9677c9a76f82 2024-11-13T18:24:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/d829ee2830584ee0b967de22452fedc4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d829ee2830584ee0b967de22452fedc4 2024-11-13T18:24:28,820 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d829ee2830584ee0b967de22452fedc4, entries=150, sequenceid=81, filesize=30.2 K 2024-11-13T18:24:28,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/f898ac289ea84bceb547496d90a28929 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f898ac289ea84bceb547496d90a28929 2024-11-13T18:24:28,834 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/C of a3e5c2fc3660c99b882b8a9ec24a558f into 955e98f1345b4cb688ac9677c9a76f82(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:28,835 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:28,835 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/C, priority=13, startTime=1731522268252; duration=0sec 2024-11-13T18:24:28,835 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:28,835 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:C 2024-11-13T18:24:28,845 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f898ac289ea84bceb547496d90a28929, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T18:24:28,856 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/6384f42ea378480b98e87d28fa6e3e3d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/6384f42ea378480b98e87d28fa6e3e3d 2024-11-13T18:24:28,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/4b18806df3f84f11854a7eddfb0ed1ab as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4b18806df3f84f11854a7eddfb0ed1ab 2024-11-13T18:24:28,871 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/B of a3e5c2fc3660c99b882b8a9ec24a558f into 6384f42ea378480b98e87d28fa6e3e3d(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:28,872 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:28,872 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/B, priority=13, startTime=1731522268250; duration=0sec 2024-11-13T18:24:28,872 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:28,872 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:B 2024-11-13T18:24:28,877 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4b18806df3f84f11854a7eddfb0ed1ab, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T18:24:28,880 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a3e5c2fc3660c99b882b8a9ec24a558f in 582ms, sequenceid=81, compaction requested=false 2024-11-13T18:24:28,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:28,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:28,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-13T18:24:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-13T18:24:28,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-13T18:24:28,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 921 msec 2024-11-13T18:24:28,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 945 msec 2024-11-13T18:24:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:29,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T18:24:29,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:29,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:29,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:29,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:29,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:29,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:24:29,118 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:29,121 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:29,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111388e06993cfcd4b1ea032086e3c8b8c90_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522268380/Put/seqid=0 2024-11-13T18:24:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-13T18:24:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:24:29,135 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:29,151 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:29,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:29,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741956_1132 (size=14594) 2024-11-13T18:24:29,178 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:29,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111388e06993cfcd4b1ea032086e3c8b8c90_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111388e06993cfcd4b1ea032086e3c8b8c90_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:29,188 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/0b70fc53254e4984848bb6c836abbcb7, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:29,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/0b70fc53254e4984848bb6c836abbcb7 is 175, key is test_row_0/A:col10/1731522268380/Put/seqid=0 2024-11-13T18:24:29,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522329176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522329183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522329207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741957_1133 (size=39549) 2024-11-13T18:24:29,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522329213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,239 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/0b70fc53254e4984848bb6c836abbcb7 2024-11-13T18:24:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:24:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522329220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/9e2ffe0c611f4a6896ed64fddd05a5b1 is 50, key is test_row_0/B:col10/1731522268380/Put/seqid=0 2024-11-13T18:24:29,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:24:29,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:29,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741958_1134 (size=12001) 2024-11-13T18:24:29,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/9e2ffe0c611f4a6896ed64fddd05a5b1 2024-11-13T18:24:29,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522329331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522329337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522329348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522329358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522329348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/d19f2fb239e5400abc7564934672f60a is 50, key is test_row_0/C:col10/1731522268380/Put/seqid=0 2024-11-13T18:24:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:24:29,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:24:29,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:29,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741959_1135 (size=12001) 2024-11-13T18:24:29,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/d19f2fb239e5400abc7564934672f60a 2024-11-13T18:24:29,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/0b70fc53254e4984848bb6c836abbcb7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/0b70fc53254e4984848bb6c836abbcb7 2024-11-13T18:24:29,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/0b70fc53254e4984848bb6c836abbcb7, entries=200, sequenceid=98, filesize=38.6 K 2024-11-13T18:24:29,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/9e2ffe0c611f4a6896ed64fddd05a5b1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9e2ffe0c611f4a6896ed64fddd05a5b1 2024-11-13T18:24:29,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9e2ffe0c611f4a6896ed64fddd05a5b1, entries=150, sequenceid=98, filesize=11.7 K 2024-11-13T18:24:29,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/d19f2fb239e5400abc7564934672f60a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d19f2fb239e5400abc7564934672f60a 2024-11-13T18:24:29,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d19f2fb239e5400abc7564934672f60a, entries=150, sequenceid=98, filesize=11.7 K 2024-11-13T18:24:29,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for a3e5c2fc3660c99b882b8a9ec24a558f in 489ms, sequenceid=98, compaction requested=true 2024-11-13T18:24:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:29,556 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:29,556 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:29,559 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:29,559 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/B is initiating minor compaction (all files) 2024-11-13T18:24:29,559 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/B in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,559 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/6384f42ea378480b98e87d28fa6e3e3d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f898ac289ea84bceb547496d90a28929, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9e2ffe0c611f4a6896ed64fddd05a5b1] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=35.3 K 2024-11-13T18:24:29,560 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 6384f42ea378480b98e87d28fa6e3e3d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522267378 2024-11-13T18:24:29,561 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:29,561 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/A is initiating minor compaction (all files) 2024-11-13T18:24:29,561 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/A in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,561 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/86098d2ca20846a4a5725d22aa222a4e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d829ee2830584ee0b967de22452fedc4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/0b70fc53254e4984848bb6c836abbcb7] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=99.2 K 2024-11-13T18:24:29,561 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,562 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/86098d2ca20846a4a5725d22aa222a4e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d829ee2830584ee0b967de22452fedc4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/0b70fc53254e4984848bb6c836abbcb7] 2024-11-13T18:24:29,562 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting f898ac289ea84bceb547496d90a28929, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731522267851 2024-11-13T18:24:29,562 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86098d2ca20846a4a5725d22aa222a4e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522267378 2024-11-13T18:24:29,563 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e2ffe0c611f4a6896ed64fddd05a5b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522268375 2024-11-13T18:24:29,563 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting d829ee2830584ee0b967de22452fedc4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731522267851 2024-11-13T18:24:29,563 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b70fc53254e4984848bb6c836abbcb7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522268344 2024-11-13T18:24:29,589 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#B#compaction#114 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:29,590 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/9a79a8fb206e430fb4b078cc5e96b6af is 50, key is test_row_0/B:col10/1731522268380/Put/seqid=0 2024-11-13T18:24:29,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:24:29,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:29,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:29,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:29,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:29,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:29,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:29,607 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:29,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:24:29,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522329637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,680 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411136a8e3fc7a7014abd810f549d2295cc11_a3e5c2fc3660c99b882b8a9ec24a558f store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522329642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522329645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,692 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411136a8e3fc7a7014abd810f549d2295cc11_a3e5c2fc3660c99b882b8a9ec24a558f, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:29,696 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136a8e3fc7a7014abd810f549d2295cc11_a3e5c2fc3660c99b882b8a9ec24a558f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:29,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522329687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741960_1136 (size=12207) 2024-11-13T18:24:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522329700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130a3ab5f9d3d545e1a81b165593918607_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522269593/Put/seqid=0 2024-11-13T18:24:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:24:29,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741961_1137 (size=4469) 2024-11-13T18:24:29,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:24:29,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:29,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,811 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#A#compaction#115 average throughput is 0.12 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,812 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/38373185977a40e8aacc8c422a0ac406 is 175, key is test_row_0/A:col10/1731522268380/Put/seqid=0 2024-11-13T18:24:29,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522329786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522329796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522329796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522329816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522329818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:29,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:29,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741962_1138 (size=14594) 2024-11-13T18:24:29,844 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:29,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130a3ab5f9d3d545e1a81b165593918607_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130a3ab5f9d3d545e1a81b165593918607_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:29,854 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/f8b1c84e28e64f4b89ea27a1e9e923d6, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:29,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/f8b1c84e28e64f4b89ea27a1e9e923d6 is 175, key is test_row_0/A:col10/1731522269593/Put/seqid=0 2024-11-13T18:24:29,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741963_1139 (size=31161) 2024-11-13T18:24:29,869 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/38373185977a40e8aacc8c422a0ac406 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/38373185977a40e8aacc8c422a0ac406 2024-11-13T18:24:29,888 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/A of a3e5c2fc3660c99b882b8a9ec24a558f into 38373185977a40e8aacc8c422a0ac406(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:29,888 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:29,888 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/A, priority=13, startTime=1731522269556; duration=0sec 2024-11-13T18:24:29,888 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:29,888 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:A 2024-11-13T18:24:29,888 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:29,895 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:29,895 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/C is initiating minor compaction (all files) 2024-11-13T18:24:29,895 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/C in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,896 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/955e98f1345b4cb688ac9677c9a76f82, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4b18806df3f84f11854a7eddfb0ed1ab, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d19f2fb239e5400abc7564934672f60a] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=35.3 K 2024-11-13T18:24:29,900 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 955e98f1345b4cb688ac9677c9a76f82, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731522267378 2024-11-13T18:24:29,901 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b18806df3f84f11854a7eddfb0ed1ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731522267851 2024-11-13T18:24:29,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741964_1140 (size=39549) 2024-11-13T18:24:29,905 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting d19f2fb239e5400abc7564934672f60a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522268375 2024-11-13T18:24:29,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/f8b1c84e28e64f4b89ea27a1e9e923d6 2024-11-13T18:24:29,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/15389914f17945a1bc73dcbb5977144b is 50, key is test_row_0/B:col10/1731522269593/Put/seqid=0 2024-11-13T18:24:29,961 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#C#compaction#118 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:29,962 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/688344bb28144b93882fac81692584ec is 50, key is test_row_0/C:col10/1731522268380/Put/seqid=0 2024-11-13T18:24:29,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:24:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:29,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:29,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:30,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741965_1141 (size=12001) 2024-11-13T18:24:30,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/15389914f17945a1bc73dcbb5977144b 2024-11-13T18:24:30,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522330026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522330026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741966_1142 (size=12207) 2024-11-13T18:24:30,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522330026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522330036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522330036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,106 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/688344bb28144b93882fac81692584ec as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/688344bb28144b93882fac81692584ec 2024-11-13T18:24:30,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:24:30,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/a6cc8dca719548a781036dc152b6972e is 50, key is test_row_0/C:col10/1731522269593/Put/seqid=0 2024-11-13T18:24:30,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:30,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:30,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:30,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:30,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:30,132 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/9a79a8fb206e430fb4b078cc5e96b6af as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9a79a8fb206e430fb4b078cc5e96b6af 2024-11-13T18:24:30,139 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/C of a3e5c2fc3660c99b882b8a9ec24a558f into 688344bb28144b93882fac81692584ec(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:30,140 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:30,142 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/C, priority=13, startTime=1731522269556; duration=0sec 2024-11-13T18:24:30,143 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:30,143 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:C 2024-11-13T18:24:30,148 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/B of a3e5c2fc3660c99b882b8a9ec24a558f into 9a79a8fb206e430fb4b078cc5e96b6af(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:30,148 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:30,149 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/B, priority=13, startTime=1731522269556; duration=0sec 2024-11-13T18:24:30,149 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:30,149 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:B 2024-11-13T18:24:30,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741967_1143 (size=12001) 2024-11-13T18:24:30,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/a6cc8dca719548a781036dc152b6972e 2024-11-13T18:24:30,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/f8b1c84e28e64f4b89ea27a1e9e923d6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/f8b1c84e28e64f4b89ea27a1e9e923d6 2024-11-13T18:24:30,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/f8b1c84e28e64f4b89ea27a1e9e923d6, entries=200, sequenceid=118, filesize=38.6 K 2024-11-13T18:24:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/15389914f17945a1bc73dcbb5977144b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/15389914f17945a1bc73dcbb5977144b 2024-11-13T18:24:30,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/15389914f17945a1bc73dcbb5977144b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T18:24:30,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/a6cc8dca719548a781036dc152b6972e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/a6cc8dca719548a781036dc152b6972e 2024-11-13T18:24:30,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/a6cc8dca719548a781036dc152b6972e, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T18:24:30,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for a3e5c2fc3660c99b882b8a9ec24a558f in 652ms, sequenceid=118, compaction requested=false 2024-11-13T18:24:30,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:24:30,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:24:30,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:30,289 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:24:30,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:30,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:30,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:30,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139e82d339b6b144618f797abd7f43c914_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522269672/Put/seqid=0 2024-11-13T18:24:30,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:30,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741968_1144 (size=12254) 2024-11-13T18:24:30,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:30,435 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139e82d339b6b144618f797abd7f43c914_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139e82d339b6b144618f797abd7f43c914_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:30,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/384c986889844aa9bbd18cb49803c9a5, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:30,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/384c986889844aa9bbd18cb49803c9a5 is 175, key is test_row_0/A:col10/1731522269672/Put/seqid=0 2024-11-13T18:24:30,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741969_1145 (size=31055) 2024-11-13T18:24:30,470 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/384c986889844aa9bbd18cb49803c9a5 2024-11-13T18:24:30,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522330457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522330483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522330485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/3b6a44e6b6084a64a99b3b5aa9d79331 is 50, key is test_row_0/B:col10/1731522269672/Put/seqid=0 2024-11-13T18:24:30,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522330487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522330489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741970_1146 (size=12101) 2024-11-13T18:24:30,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522330596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522330611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522330617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522330617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522330629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522330807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522330847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522330847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522330847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522330867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:30,982 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/3b6a44e6b6084a64a99b3b5aa9d79331 2024-11-13T18:24:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/6adcf2fe3a1c4c8c926f3556260e4fb6 is 50, key is test_row_0/C:col10/1731522269672/Put/seqid=0 2024-11-13T18:24:31,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741971_1147 (size=12101) 2024-11-13T18:24:31,060 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/6adcf2fe3a1c4c8c926f3556260e4fb6 2024-11-13T18:24:31,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/384c986889844aa9bbd18cb49803c9a5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/384c986889844aa9bbd18cb49803c9a5 2024-11-13T18:24:31,077 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/384c986889844aa9bbd18cb49803c9a5, entries=150, sequenceid=134, filesize=30.3 K 2024-11-13T18:24:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/3b6a44e6b6084a64a99b3b5aa9d79331 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/3b6a44e6b6084a64a99b3b5aa9d79331 2024-11-13T18:24:31,096 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/3b6a44e6b6084a64a99b3b5aa9d79331, entries=150, sequenceid=134, filesize=11.8 K 2024-11-13T18:24:31,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/6adcf2fe3a1c4c8c926f3556260e4fb6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/6adcf2fe3a1c4c8c926f3556260e4fb6 2024-11-13T18:24:31,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522331127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,164 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/6adcf2fe3a1c4c8c926f3556260e4fb6, entries=150, sequenceid=134, filesize=11.8 K 2024-11-13T18:24:31,165 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a3e5c2fc3660c99b882b8a9ec24a558f in 876ms, sequenceid=134, compaction requested=true 2024-11-13T18:24:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-13T18:24:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-13T18:24:31,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-13T18:24:31,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0170 sec 2024-11-13T18:24:31,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.0820 sec 2024-11-13T18:24:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:31,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:24:31,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522270452, max timestamp=1731522271205Num uniques 859; 2024-11-13T18:24:31,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:31,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:31,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:31,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:31,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:31,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:31,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:31,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130e05540494194dd08fc452c9af86c836_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522271205/Put/seqid=0 2024-11-13T18:24:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:24:31,276 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:31,279 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-13T18:24:31,300 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:24:31,301 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:31,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:31,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741972_1148 (size=12304) 2024-11-13T18:24:31,319 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:31,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130e05540494194dd08fc452c9af86c836_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130e05540494194dd08fc452c9af86c836_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:31,335 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/39aeec3a31b941cf9a2f8d43de049006, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:31,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/39aeec3a31b941cf9a2f8d43de049006 is 175, key is test_row_0/A:col10/1731522271205/Put/seqid=0 2024-11-13T18:24:31,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522331371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741973_1149 (size=31105) 2024-11-13T18:24:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:24:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522331386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522331374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522331387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T18:24:31,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:31,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:31,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:31,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:31,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522331516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522331516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522331506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522331525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:24:31,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T18:24:31,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:31,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:31,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:31,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:31,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:31,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522331659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522331753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522331756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522331757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522331758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:31,815 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/39aeec3a31b941cf9a2f8d43de049006 2024-11-13T18:24:31,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T18:24:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:31,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:31,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/c64e08d752014121b303d1a6e3a2f614 is 50, key is test_row_0/B:col10/1731522271205/Put/seqid=0 2024-11-13T18:24:31,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741974_1150 (size=12151) 2024-11-13T18:24:31,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/c64e08d752014121b303d1a6e3a2f614 2024-11-13T18:24:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:24:31,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/ba484afefb7643de8d8f1f1338f23f2e is 50, key is test_row_0/C:col10/1731522271205/Put/seqid=0 2024-11-13T18:24:31,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T18:24:32,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:32,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:32,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:32,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:32,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:32,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741975_1151 (size=12151) 2024-11-13T18:24:32,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/ba484afefb7643de8d8f1f1338f23f2e 2024-11-13T18:24:32,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/39aeec3a31b941cf9a2f8d43de049006 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/39aeec3a31b941cf9a2f8d43de049006 2024-11-13T18:24:32,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/39aeec3a31b941cf9a2f8d43de049006, entries=150, sequenceid=155, filesize=30.4 K 2024-11-13T18:24:32,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522332076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522332076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522332076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522332076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/c64e08d752014121b303d1a6e3a2f614 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c64e08d752014121b303d1a6e3a2f614 2024-11-13T18:24:32,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c64e08d752014121b303d1a6e3a2f614, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T18:24:32,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/ba484afefb7643de8d8f1f1338f23f2e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ba484afefb7643de8d8f1f1338f23f2e 2024-11-13T18:24:32,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T18:24:32,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:32,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:32,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:32,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:32,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:32,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ba484afefb7643de8d8f1f1338f23f2e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T18:24:32,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for a3e5c2fc3660c99b882b8a9ec24a558f in 964ms, sequenceid=155, compaction requested=true 2024-11-13T18:24:32,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:32,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:32,175 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:24:32,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:32,176 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:24:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:32,192 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132870 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:24:32,192 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/A is initiating minor compaction (all files) 2024-11-13T18:24:32,192 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/A in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:32,192 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/38373185977a40e8aacc8c422a0ac406, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/f8b1c84e28e64f4b89ea27a1e9e923d6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/384c986889844aa9bbd18cb49803c9a5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/39aeec3a31b941cf9a2f8d43de049006] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=129.8 K 2024-11-13T18:24:32,193 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:32,193 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/38373185977a40e8aacc8c422a0ac406, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/f8b1c84e28e64f4b89ea27a1e9e923d6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/384c986889844aa9bbd18cb49803c9a5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/39aeec3a31b941cf9a2f8d43de049006] 2024-11-13T18:24:32,195 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38373185977a40e8aacc8c422a0ac406, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522268375 2024-11-13T18:24:32,195 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:24:32,195 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/B is initiating minor compaction (all files) 2024-11-13T18:24:32,196 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/B in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:32,196 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9a79a8fb206e430fb4b078cc5e96b6af, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/15389914f17945a1bc73dcbb5977144b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/3b6a44e6b6084a64a99b3b5aa9d79331, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c64e08d752014121b303d1a6e3a2f614] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=47.3 K 2024-11-13T18:24:32,197 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a79a8fb206e430fb4b078cc5e96b6af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522268375 2024-11-13T18:24:32,198 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8b1c84e28e64f4b89ea27a1e9e923d6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731522269200 2024-11-13T18:24:32,203 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 15389914f17945a1bc73dcbb5977144b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731522269200 2024-11-13T18:24:32,204 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b6a44e6b6084a64a99b3b5aa9d79331, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731522269642 2024-11-13T18:24:32,204 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384c986889844aa9bbd18cb49803c9a5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731522269642 2024-11-13T18:24:32,207 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c64e08d752014121b303d1a6e3a2f614, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731522270476 2024-11-13T18:24:32,207 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39aeec3a31b941cf9a2f8d43de049006, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731522270476 2024-11-13T18:24:32,266 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#B#compaction#126 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:32,268 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/493ccd447b9649ebba2caffdb18d38d7 is 50, key is test_row_0/B:col10/1731522271205/Put/seqid=0 2024-11-13T18:24:32,280 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:32,294 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411130f4a8b1381bf4b91b5c970af20c74718_a3e5c2fc3660c99b882b8a9ec24a558f store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:32,302 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411130f4a8b1381bf4b91b5c970af20c74718_a3e5c2fc3660c99b882b8a9ec24a558f, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:32,302 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130f4a8b1381bf4b91b5c970af20c74718_a3e5c2fc3660c99b882b8a9ec24a558f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:32,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T18:24:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:32,317 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:24:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:32,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741976_1152 (size=12493) 2024-11-13T18:24:32,350 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/493ccd447b9649ebba2caffdb18d38d7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/493ccd447b9649ebba2caffdb18d38d7 2024-11-13T18:24:32,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111379d8ea1bda214b608a19a6aa26364290_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522271316/Put/seqid=0 2024-11-13T18:24:32,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741977_1153 (size=4469) 2024-11-13T18:24:32,373 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/B of a3e5c2fc3660c99b882b8a9ec24a558f into 493ccd447b9649ebba2caffdb18d38d7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:32,374 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:32,374 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/B, priority=12, startTime=1731522272176; duration=0sec 2024-11-13T18:24:32,374 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:32,374 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:B 2024-11-13T18:24:32,374 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:24:32,378 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#A#compaction#127 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:32,378 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/80b3c4fbca75431e973b310a5deb2b7c is 175, key is test_row_0/A:col10/1731522271205/Put/seqid=0 2024-11-13T18:24:32,381 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:24:32,382 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/C is initiating minor compaction (all files) 2024-11-13T18:24:32,382 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/C in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:32,383 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/688344bb28144b93882fac81692584ec, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/a6cc8dca719548a781036dc152b6972e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/6adcf2fe3a1c4c8c926f3556260e4fb6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ba484afefb7643de8d8f1f1338f23f2e] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=47.3 K 2024-11-13T18:24:32,385 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 688344bb28144b93882fac81692584ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731522268375 2024-11-13T18:24:32,391 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting a6cc8dca719548a781036dc152b6972e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731522269200 2024-11-13T18:24:32,392 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 6adcf2fe3a1c4c8c926f3556260e4fb6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731522269642 2024-11-13T18:24:32,392 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting ba484afefb7643de8d8f1f1338f23f2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731522270476 2024-11-13T18:24:32,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741978_1154 (size=12304) 2024-11-13T18:24:32,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:32,459 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111379d8ea1bda214b608a19a6aa26364290_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111379d8ea1bda214b608a19a6aa26364290_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:32,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/026582010f95475fb2371b0107e66b66, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:32,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/026582010f95475fb2371b0107e66b66 is 175, key is test_row_0/A:col10/1731522271316/Put/seqid=0 2024-11-13T18:24:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:24:32,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741979_1155 (size=31447) 2024-11-13T18:24:32,484 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/80b3c4fbca75431e973b310a5deb2b7c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/80b3c4fbca75431e973b310a5deb2b7c 2024-11-13T18:24:32,490 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#C#compaction#129 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:32,491 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/cc375279685848e0aabb0ce5494e3ec3 is 50, key is test_row_0/C:col10/1731522271205/Put/seqid=0 2024-11-13T18:24:32,515 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/A of a3e5c2fc3660c99b882b8a9ec24a558f into 80b3c4fbca75431e973b310a5deb2b7c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:32,515 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:32,515 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/A, priority=12, startTime=1731522272175; duration=0sec 2024-11-13T18:24:32,515 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:32,515 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:A 2024-11-13T18:24:32,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741980_1156 (size=31105) 2024-11-13T18:24:32,585 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/026582010f95475fb2371b0107e66b66 2024-11-13T18:24:32,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741981_1157 (size=12493) 2024-11-13T18:24:32,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:32,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/82c8b7cfa00a4d0c9c28bce36d3b443e is 50, key is test_row_0/B:col10/1731522271316/Put/seqid=0 2024-11-13T18:24:32,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741982_1158 (size=12151) 2024-11-13T18:24:32,697 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/82c8b7cfa00a4d0c9c28bce36d3b443e 2024-11-13T18:24:32,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522332692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522332695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522332695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,732 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2275 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/7e5f28a154aa4fa2bc82c81bb36d3c6f is 50, key is test_row_0/C:col10/1731522271316/Put/seqid=0 2024-11-13T18:24:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522332696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522332696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741983_1159 (size=12151) 2024-11-13T18:24:32,807 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/7e5f28a154aa4fa2bc82c81bb36d3c6f 2024-11-13T18:24:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/026582010f95475fb2371b0107e66b66 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/026582010f95475fb2371b0107e66b66 2024-11-13T18:24:32,831 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/026582010f95475fb2371b0107e66b66, entries=150, sequenceid=171, filesize=30.4 K 2024-11-13T18:24:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/82c8b7cfa00a4d0c9c28bce36d3b443e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/82c8b7cfa00a4d0c9c28bce36d3b443e 2024-11-13T18:24:32,850 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/82c8b7cfa00a4d0c9c28bce36d3b443e, entries=150, sequenceid=171, filesize=11.9 K 2024-11-13T18:24:32,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/7e5f28a154aa4fa2bc82c81bb36d3c6f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/7e5f28a154aa4fa2bc82c81bb36d3c6f 2024-11-13T18:24:32,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522332851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,863 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/7e5f28a154aa4fa2bc82c81bb36d3c6f, entries=150, sequenceid=171, filesize=11.9 K 2024-11-13T18:24:32,865 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a3e5c2fc3660c99b882b8a9ec24a558f in 548ms, sequenceid=171, compaction requested=false 2024-11-13T18:24:32,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:32,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:32,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-13T18:24:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-13T18:24:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-13T18:24:32,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-11-13T18:24:32,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:24:32,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:32,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:32,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:32,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:32,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:32,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:32,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.5940 sec 2024-11-13T18:24:32,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111364f1e6fcf3cf431b91acec4a70bad32f_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522272685/Put/seqid=0 2024-11-13T18:24:32,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741984_1160 (size=14794) 2024-11-13T18:24:32,970 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522332965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:32,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111364f1e6fcf3cf431b91acec4a70bad32f_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111364f1e6fcf3cf431b91acec4a70bad32f_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:32,985 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/9ee2adff2bc142adb6da460a34c4fd50, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:32,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/9ee2adff2bc142adb6da460a34c4fd50 is 175, key is test_row_0/A:col10/1731522272685/Put/seqid=0 2024-11-13T18:24:33,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522332977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522332983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,023 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/cc375279685848e0aabb0ce5494e3ec3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cc375279685848e0aabb0ce5494e3ec3 2024-11-13T18:24:33,039 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/C of a3e5c2fc3660c99b882b8a9ec24a558f into cc375279685848e0aabb0ce5494e3ec3(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:33,039 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:33,039 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/C, priority=12, startTime=1731522272189; duration=0sec 2024-11-13T18:24:33,039 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:33,039 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:C 2024-11-13T18:24:33,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741985_1161 (size=39749) 2024-11-13T18:24:33,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522333082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522333090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522333119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522333132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522333319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522333347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522333360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522333403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,447 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/9ee2adff2bc142adb6da460a34c4fd50 2024-11-13T18:24:33,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/a832e99a68a74a43975d9f83ee8f9464 is 50, key is test_row_0/B:col10/1731522272685/Put/seqid=0 2024-11-13T18:24:33,478 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:24:33,479 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T18:24:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:24:33,487 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:33,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-13T18:24:33,491 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:24:33,493 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:33,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:33,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741986_1162 (size=12151) 2024-11-13T18:24:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:24:33,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:24:33,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:33,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:33,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522333646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522333664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522333675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:24:33,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:33,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:33,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:33,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:33,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:24:33,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522333923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:33,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/a832e99a68a74a43975d9f83ee8f9464 2024-11-13T18:24:33,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:24:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:33,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:34,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/ff862b59f62e482cbc700d7cbfa455a0 is 50, key is test_row_0/C:col10/1731522272685/Put/seqid=0 2024-11-13T18:24:34,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741987_1163 (size=12151) 2024-11-13T18:24:34,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:24:34,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:24:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:34,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522334167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522334187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522334198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:34,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:34,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:34,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:34,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:24:34,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:34,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:34,359 INFO [master/62ae331128c9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T18:24:34,359 INFO [master/62ae331128c9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T18:24:34,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:24:34,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:34,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/ff862b59f62e482cbc700d7cbfa455a0 2024-11-13T18:24:34,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/9ee2adff2bc142adb6da460a34c4fd50 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9ee2adff2bc142adb6da460a34c4fd50 2024-11-13T18:24:34,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9ee2adff2bc142adb6da460a34c4fd50, entries=200, sequenceid=190, filesize=38.8 K 2024-11-13T18:24:34,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/a832e99a68a74a43975d9f83ee8f9464 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/a832e99a68a74a43975d9f83ee8f9464 2024-11-13T18:24:34,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/a832e99a68a74a43975d9f83ee8f9464, entries=150, sequenceid=190, filesize=11.9 K 2024-11-13T18:24:34,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/ff862b59f62e482cbc700d7cbfa455a0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ff862b59f62e482cbc700d7cbfa455a0 2024-11-13T18:24:34,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ff862b59f62e482cbc700d7cbfa455a0, entries=150, sequenceid=190, filesize=11.9 K 2024-11-13T18:24:34,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a3e5c2fc3660c99b882b8a9ec24a558f in 1680ms, sequenceid=190, compaction requested=true 2024-11-13T18:24:34,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:34,554 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:34,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:34,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:34,555 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:34,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:34,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:34,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:34,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:34,564 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:34,564 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/A is initiating minor compaction (all files) 2024-11-13T18:24:34,564 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/A in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,565 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/80b3c4fbca75431e973b310a5deb2b7c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/026582010f95475fb2371b0107e66b66, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9ee2adff2bc142adb6da460a34c4fd50] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=99.9 K 2024-11-13T18:24:34,565 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,565 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/80b3c4fbca75431e973b310a5deb2b7c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/026582010f95475fb2371b0107e66b66, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9ee2adff2bc142adb6da460a34c4fd50] 2024-11-13T18:24:34,571 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:34,571 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/B is initiating minor compaction (all files) 2024-11-13T18:24:34,571 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/B in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,571 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/493ccd447b9649ebba2caffdb18d38d7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/82c8b7cfa00a4d0c9c28bce36d3b443e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/a832e99a68a74a43975d9f83ee8f9464] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=35.9 K 2024-11-13T18:24:34,572 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b3c4fbca75431e973b310a5deb2b7c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731522270476 2024-11-13T18:24:34,572 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 493ccd447b9649ebba2caffdb18d38d7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731522270476 2024-11-13T18:24:34,573 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 82c8b7cfa00a4d0c9c28bce36d3b443e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731522271316 2024-11-13T18:24:34,573 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting a832e99a68a74a43975d9f83ee8f9464, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731522272685 2024-11-13T18:24:34,578 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 026582010f95475fb2371b0107e66b66, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731522271316 2024-11-13T18:24:34,583 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ee2adff2bc142adb6da460a34c4fd50, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731522272676 2024-11-13T18:24:34,605 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:34,606 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/e84ca77978b445b1b7bec18f77546f57 is 50, key is test_row_0/B:col10/1731522272685/Put/seqid=0 2024-11-13T18:24:34,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:24:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,616 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:24:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:34,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:34,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:34,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:34,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:24:34,627 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:34,655 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111309451e716cda464b91b9c29e36fbc10c_a3e5c2fc3660c99b882b8a9ec24a558f store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:34,658 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111309451e716cda464b91b9c29e36fbc10c_a3e5c2fc3660c99b882b8a9ec24a558f, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:34,659 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111309451e716cda464b91b9c29e36fbc10c_a3e5c2fc3660c99b882b8a9ec24a558f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:34,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741988_1164 (size=12595) 2024-11-13T18:24:34,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132ebb84fd95704e3683e976cc50f8d734_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522272969/Put/seqid=0 2024-11-13T18:24:34,710 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/e84ca77978b445b1b7bec18f77546f57 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/e84ca77978b445b1b7bec18f77546f57 2024-11-13T18:24:34,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741989_1165 (size=4469) 2024-11-13T18:24:34,738 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#A#compaction#136 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:34,739 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/d6e9ab6b7000425187bdac8a98bee0d8 is 175, key is test_row_0/A:col10/1731522272685/Put/seqid=0 2024-11-13T18:24:34,746 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/B of a3e5c2fc3660c99b882b8a9ec24a558f into e84ca77978b445b1b7bec18f77546f57(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:34,746 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:34,746 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/B, priority=13, startTime=1731522274554; duration=0sec 2024-11-13T18:24:34,746 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:34,746 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:B 2024-11-13T18:24:34,746 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:34,749 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:34,749 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/C is initiating minor compaction (all files) 2024-11-13T18:24:34,749 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/C in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:34,749 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cc375279685848e0aabb0ce5494e3ec3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/7e5f28a154aa4fa2bc82c81bb36d3c6f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ff862b59f62e482cbc700d7cbfa455a0] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=35.9 K 2024-11-13T18:24:34,750 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting cc375279685848e0aabb0ce5494e3ec3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731522270476 2024-11-13T18:24:34,750 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5f28a154aa4fa2bc82c81bb36d3c6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731522271316 2024-11-13T18:24:34,751 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting ff862b59f62e482cbc700d7cbfa455a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731522272685 2024-11-13T18:24:34,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741990_1166 (size=12304) 2024-11-13T18:24:34,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:34,783 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132ebb84fd95704e3683e976cc50f8d734_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132ebb84fd95704e3683e976cc50f8d734_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:34,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741991_1167 (size=31549) 2024-11-13T18:24:34,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/db21fb9913fa4d83be4e9ea6be695fa2, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:34,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/db21fb9913fa4d83be4e9ea6be695fa2 is 175, key is test_row_0/A:col10/1731522272969/Put/seqid=0 2024-11-13T18:24:34,803 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:34,804 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/3255881018034544bef39f33fff641f6 is 50, key is test_row_0/C:col10/1731522272685/Put/seqid=0 2024-11-13T18:24:34,811 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/d6e9ab6b7000425187bdac8a98bee0d8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d6e9ab6b7000425187bdac8a98bee0d8 2024-11-13T18:24:34,825 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/A of a3e5c2fc3660c99b882b8a9ec24a558f into d6e9ab6b7000425187bdac8a98bee0d8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:34,825 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:34,825 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/A, priority=13, startTime=1731522274554; duration=0sec 2024-11-13T18:24:34,826 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:34,826 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:A 2024-11-13T18:24:34,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741992_1168 (size=31105) 2024-11-13T18:24:34,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741993_1169 (size=12595) 2024-11-13T18:24:34,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522334935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:34,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522334947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:34,957 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:34,971 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/3255881018034544bef39f33fff641f6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/3255881018034544bef39f33fff641f6 2024-11-13T18:24:34,981 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/C of a3e5c2fc3660c99b882b8a9ec24a558f into 3255881018034544bef39f33fff641f6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:34,981 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:34,981 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/C, priority=13, startTime=1731522274556; duration=0sec 2024-11-13T18:24:34,981 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:34,981 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:C 2024-11-13T18:24:35,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522335063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522335199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:35,211 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:35,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522335212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:35,220 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_1' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:35,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:35,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522335228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:35,231 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:35,256 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/db21fb9913fa4d83be4e9ea6be695fa2 2024-11-13T18:24:35,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/c216beb03f3443c7bb5dc72d72ddec15 is 50, key is test_row_0/B:col10/1731522272969/Put/seqid=0 2024-11-13T18:24:35,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522335287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:35,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741994_1170 (size=12151) 2024-11-13T18:24:35,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522335607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:24:35,768 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/c216beb03f3443c7bb5dc72d72ddec15 2024-11-13T18:24:35,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/1e7af5cb78e64da5963334fd459ea872 is 50, key is test_row_0/C:col10/1731522272969/Put/seqid=0 2024-11-13T18:24:35,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741995_1171 (size=12151) 2024-11-13T18:24:35,852 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/1e7af5cb78e64da5963334fd459ea872 2024-11-13T18:24:35,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/db21fb9913fa4d83be4e9ea6be695fa2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/db21fb9913fa4d83be4e9ea6be695fa2 2024-11-13T18:24:35,876 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/db21fb9913fa4d83be4e9ea6be695fa2, entries=150, sequenceid=206, filesize=30.4 K 2024-11-13T18:24:35,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/c216beb03f3443c7bb5dc72d72ddec15 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c216beb03f3443c7bb5dc72d72ddec15 2024-11-13T18:24:35,919 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c216beb03f3443c7bb5dc72d72ddec15, entries=150, sequenceid=206, filesize=11.9 K 2024-11-13T18:24:35,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/1e7af5cb78e64da5963334fd459ea872 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/1e7af5cb78e64da5963334fd459ea872 2024-11-13T18:24:35,940 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/1e7af5cb78e64da5963334fd459ea872, entries=150, sequenceid=206, filesize=11.9 K 2024-11-13T18:24:35,943 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a3e5c2fc3660c99b882b8a9ec24a558f in 1327ms, sequenceid=206, compaction requested=false 2024-11-13T18:24:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-13T18:24:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-13T18:24:35,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-13T18:24:35,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4530 sec 2024-11-13T18:24:35,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.4790 sec 2024-11-13T18:24:36,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:24:36,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:36,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111359630d9c17ef4a87a1f23b62d93e713c_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522276147/Put/seqid=0 2024-11-13T18:24:36,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741996_1172 (size=14794) 2024-11-13T18:24:36,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522336354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:36,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522336467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:36,655 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:36,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111359630d9c17ef4a87a1f23b62d93e713c_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111359630d9c17ef4a87a1f23b62d93e713c_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:36,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/11550e7368b4442b883533b07483a959, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:36,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/11550e7368b4442b883533b07483a959 is 175, key is test_row_0/A:col10/1731522276147/Put/seqid=0 2024-11-13T18:24:36,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522336688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:36,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:36,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:36,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:36,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741997_1173 (size=39749) 2024-11-13T18:24:36,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522336979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:36,999 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4307 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:37,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522337006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:37,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/11550e7368b4442b883533b07483a959 2024-11-13T18:24:37,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/801ac13f664549928839c21393543e42 is 50, key is test_row_0/B:col10/1731522276147/Put/seqid=0 2024-11-13T18:24:37,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741998_1174 (size=12151) 2024-11-13T18:24:37,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522337230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:37,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522337231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:37,248 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_1' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:37,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522337250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:37,252 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:37,255 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:37,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522337525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:37,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/801ac13f664549928839c21393543e42 2024-11-13T18:24:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:24:37,667 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:37,671 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-13T18:24:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:24:37,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/d1d2379cc191406385a0b765165566d1 is 50, key is test_row_0/C:col10/1731522276147/Put/seqid=0 2024-11-13T18:24:37,688 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:37,696 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:37,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:37,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741999_1175 (size=12151) 2024-11-13T18:24:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:24:37,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T18:24:37,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:37,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:37,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:37,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:37,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:24:38,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T18:24:38,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:38,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:38,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:38,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:38,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:38,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/d1d2379cc191406385a0b765165566d1 2024-11-13T18:24:38,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/11550e7368b4442b883533b07483a959 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11550e7368b4442b883533b07483a959 2024-11-13T18:24:38,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T18:24:38,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:38,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:38,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:38,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:38,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:38,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11550e7368b4442b883533b07483a959, entries=200, sequenceid=227, filesize=38.8 K 2024-11-13T18:24:38,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/801ac13f664549928839c21393543e42 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/801ac13f664549928839c21393543e42 2024-11-13T18:24:38,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/801ac13f664549928839c21393543e42, entries=150, sequenceid=227, filesize=11.9 K 2024-11-13T18:24:38,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/d1d2379cc191406385a0b765165566d1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d1d2379cc191406385a0b765165566d1 2024-11-13T18:24:38,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d1d2379cc191406385a0b765165566d1, entries=150, sequenceid=227, filesize=11.9 K 2024-11-13T18:24:38,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a3e5c2fc3660c99b882b8a9ec24a558f in 2103ms, sequenceid=227, compaction requested=true 2024-11-13T18:24:38,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:38,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:38,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:38,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:38,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:24:38,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:38,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:24:38,289 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:38,290 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:38,298 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:38,299 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/A is initiating minor compaction (all files) 2024-11-13T18:24:38,299 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/A in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:38,299 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d6e9ab6b7000425187bdac8a98bee0d8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/db21fb9913fa4d83be4e9ea6be695fa2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11550e7368b4442b883533b07483a959] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=100.0 K 2024-11-13T18:24:38,299 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:38,299 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d6e9ab6b7000425187bdac8a98bee0d8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/db21fb9913fa4d83be4e9ea6be695fa2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11550e7368b4442b883533b07483a959] 2024-11-13T18:24:38,304 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e9ab6b7000425187bdac8a98bee0d8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731522272685 2024-11-13T18:24:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:24:38,311 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:38,311 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/C is initiating minor compaction (all files) 2024-11-13T18:24:38,311 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/C in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:38,311 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/3255881018034544bef39f33fff641f6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/1e7af5cb78e64da5963334fd459ea872, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d1d2379cc191406385a0b765165566d1] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=36.0 K 2024-11-13T18:24:38,312 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting db21fb9913fa4d83be4e9ea6be695fa2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731522272928 2024-11-13T18:24:38,315 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3255881018034544bef39f33fff641f6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731522272685 2024-11-13T18:24:38,315 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 11550e7368b4442b883533b07483a959, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731522274800 2024-11-13T18:24:38,317 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e7af5cb78e64da5963334fd459ea872, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731522272928 2024-11-13T18:24:38,323 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1d2379cc191406385a0b765165566d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731522274800 2024-11-13T18:24:38,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T18:24:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:38,354 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T18:24:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:38,378 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:38,405 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#C#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:38,405 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/67010b0e81ff47ba8d94fe237c11f2e1 is 50, key is test_row_0/C:col10/1731522276147/Put/seqid=0 2024-11-13T18:24:38,411 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111322bb6b2f17c746beaa35daffc06c253c_a3e5c2fc3660c99b882b8a9ec24a558f store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:38,419 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111322bb6b2f17c746beaa35daffc06c253c_a3e5c2fc3660c99b882b8a9ec24a558f, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:38,419 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111322bb6b2f17c746beaa35daffc06c253c_a3e5c2fc3660c99b882b8a9ec24a558f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:38,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113253730d981a24104af1468c301ed679b_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522276340/Put/seqid=0 2024-11-13T18:24:38,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742000_1176 (size=12697) 2024-11-13T18:24:38,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742001_1177 (size=12304) 2024-11-13T18:24:38,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:38,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742002_1178 (size=4469) 2024-11-13T18:24:38,500 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113253730d981a24104af1468c301ed679b_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113253730d981a24104af1468c301ed679b_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:38,501 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/67010b0e81ff47ba8d94fe237c11f2e1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/67010b0e81ff47ba8d94fe237c11f2e1 2024-11-13T18:24:38,502 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#A#compaction#144 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:38,503 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/11bce6e5d5984346a58235e47e0427f9 is 175, key is test_row_0/A:col10/1731522276147/Put/seqid=0 2024-11-13T18:24:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/ef5c69d62d8f457a8824f3849e5867f5, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:38,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/ef5c69d62d8f457a8824f3849e5867f5 is 175, key is test_row_0/A:col10/1731522276340/Put/seqid=0 2024-11-13T18:24:38,531 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/C of a3e5c2fc3660c99b882b8a9ec24a558f into 67010b0e81ff47ba8d94fe237c11f2e1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:38,531 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:38,531 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/C, priority=13, startTime=1731522278287; duration=0sec 2024-11-13T18:24:38,532 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:38,532 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:C 2024-11-13T18:24:38,532 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:38,535 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:38,535 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/B is initiating minor compaction (all files) 2024-11-13T18:24:38,535 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/B in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:38,535 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/e84ca77978b445b1b7bec18f77546f57, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c216beb03f3443c7bb5dc72d72ddec15, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/801ac13f664549928839c21393543e42] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=36.0 K 2024-11-13T18:24:38,538 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting e84ca77978b445b1b7bec18f77546f57, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731522272685 2024-11-13T18:24:38,538 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting c216beb03f3443c7bb5dc72d72ddec15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731522272928 2024-11-13T18:24:38,542 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 801ac13f664549928839c21393543e42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731522274800 2024-11-13T18:24:38,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:38,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742004_1180 (size=31105) 2024-11-13T18:24:38,559 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/ef5c69d62d8f457a8824f3849e5867f5 2024-11-13T18:24:38,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742003_1179 (size=31651) 2024-11-13T18:24:38,601 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:38,601 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/b5d761ee33e64577a2b99308c26b7d70 is 50, key is test_row_0/B:col10/1731522276147/Put/seqid=0 2024-11-13T18:24:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/45f73acbb71144929af6fd1b3eb12d00 is 50, key is test_row_0/B:col10/1731522276340/Put/seqid=0 2024-11-13T18:24:38,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742005_1181 (size=12697) 2024-11-13T18:24:38,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742006_1182 (size=12151) 2024-11-13T18:24:38,659 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/45f73acbb71144929af6fd1b3eb12d00 2024-11-13T18:24:38,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/df5e2fbe4ab24af9b2a44eef3d258d75 is 50, key is test_row_0/C:col10/1731522276340/Put/seqid=0 2024-11-13T18:24:38,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742007_1183 (size=12151) 2024-11-13T18:24:38,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522338785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:38,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:38,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:38,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:24:38,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522338906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:38,975 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/11bce6e5d5984346a58235e47e0427f9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11bce6e5d5984346a58235e47e0427f9 2024-11-13T18:24:38,991 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/A of a3e5c2fc3660c99b882b8a9ec24a558f into 11bce6e5d5984346a58235e47e0427f9(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:38,991 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:38,991 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/A, priority=13, startTime=1731522278286; duration=0sec 2024-11-13T18:24:38,993 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:38,993 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:A 2024-11-13T18:24:39,071 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/b5d761ee33e64577a2b99308c26b7d70 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/b5d761ee33e64577a2b99308c26b7d70 2024-11-13T18:24:39,081 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/B of a3e5c2fc3660c99b882b8a9ec24a558f into b5d761ee33e64577a2b99308c26b7d70(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:39,081 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:39,081 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/B, priority=13, startTime=1731522278287; duration=0sec 2024-11-13T18:24:39,083 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:39,083 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:B 2024-11-13T18:24:39,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522339119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:39,138 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/df5e2fbe4ab24af9b2a44eef3d258d75 2024-11-13T18:24:39,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/ef5c69d62d8f457a8824f3849e5867f5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/ef5c69d62d8f457a8824f3849e5867f5 2024-11-13T18:24:39,170 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/ef5c69d62d8f457a8824f3849e5867f5, entries=150, sequenceid=241, filesize=30.4 K 2024-11-13T18:24:39,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/45f73acbb71144929af6fd1b3eb12d00 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/45f73acbb71144929af6fd1b3eb12d00 2024-11-13T18:24:39,228 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/45f73acbb71144929af6fd1b3eb12d00, entries=150, sequenceid=241, filesize=11.9 K 2024-11-13T18:24:39,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/df5e2fbe4ab24af9b2a44eef3d258d75 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/df5e2fbe4ab24af9b2a44eef3d258d75 2024-11-13T18:24:39,245 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/df5e2fbe4ab24af9b2a44eef3d258d75, entries=150, sequenceid=241, filesize=11.9 K 2024-11-13T18:24:39,246 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a3e5c2fc3660c99b882b8a9ec24a558f in 892ms, sequenceid=241, compaction requested=false 2024-11-13T18:24:39,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:39,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:39,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-13T18:24:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-13T18:24:39,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-13T18:24:39,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5630 sec 2024-11-13T18:24:39,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6280 sec 2024-11-13T18:24:39,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:24:39,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:39,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:39,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:39,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:39,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:39,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:39,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113faec04e2cd104740bd9d19885322af07_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522278747/Put/seqid=0 2024-11-13T18:24:39,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742008_1184 (size=12454) 2024-11-13T18:24:39,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:39,504 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113faec04e2cd104740bd9d19885322af07_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113faec04e2cd104740bd9d19885322af07_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:39,509 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/6b39f37f5f3644d1bf59c1af70ccf8c2, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:39,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/6b39f37f5f3644d1bf59c1af70ccf8c2 is 175, key is test_row_0/A:col10/1731522278747/Put/seqid=0 2024-11-13T18:24:39,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742009_1185 (size=31255) 2024-11-13T18:24:39,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/6b39f37f5f3644d1bf59c1af70ccf8c2 2024-11-13T18:24:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522339556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:39,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/bfc597fd28c94a4fa957aae7e5877669 is 50, key is test_row_0/B:col10/1731522278747/Put/seqid=0 2024-11-13T18:24:39,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742010_1186 (size=12301) 2024-11-13T18:24:39,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522339667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:39,692 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:24:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:24:39,838 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:39,841 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-13T18:24:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:24:39,845 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:39,846 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:39,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:39,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522339887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:24:40,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:24:40,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:40,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:40,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:40,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:40,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:40,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/bfc597fd28c94a4fa957aae7e5877669 2024-11-13T18:24:40,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/cde1eacd1105472e9f51279aea2b12b0 is 50, key is test_row_0/C:col10/1731522278747/Put/seqid=0 2024-11-13T18:24:40,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742011_1187 (size=12301) 2024-11-13T18:24:40,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/cde1eacd1105472e9f51279aea2b12b0 2024-11-13T18:24:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:24:40,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/6b39f37f5f3644d1bf59c1af70ccf8c2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/6b39f37f5f3644d1bf59c1af70ccf8c2 2024-11-13T18:24:40,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:24:40,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:40,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:40,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:40,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:40,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:40,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/6b39f37f5f3644d1bf59c1af70ccf8c2, entries=150, sequenceid=263, filesize=30.5 K 2024-11-13T18:24:40,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/bfc597fd28c94a4fa957aae7e5877669 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/bfc597fd28c94a4fa957aae7e5877669 2024-11-13T18:24:40,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522340201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:40,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/bfc597fd28c94a4fa957aae7e5877669, entries=150, sequenceid=263, filesize=12.0 K 2024-11-13T18:24:40,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/cde1eacd1105472e9f51279aea2b12b0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cde1eacd1105472e9f51279aea2b12b0 2024-11-13T18:24:40,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cde1eacd1105472e9f51279aea2b12b0, entries=150, sequenceid=263, filesize=12.0 K 2024-11-13T18:24:40,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a3e5c2fc3660c99b882b8a9ec24a558f in 845ms, sequenceid=263, compaction requested=true 2024-11-13T18:24:40,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:40,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:40,287 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:40,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:40,287 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:40,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:40,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:40,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:40,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:40,303 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:40,303 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/A is initiating minor compaction (all files) 2024-11-13T18:24:40,303 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/A in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:40,303 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11bce6e5d5984346a58235e47e0427f9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/ef5c69d62d8f457a8824f3849e5867f5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/6b39f37f5f3644d1bf59c1af70ccf8c2] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=91.8 K 2024-11-13T18:24:40,303 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:40,303 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11bce6e5d5984346a58235e47e0427f9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/ef5c69d62d8f457a8824f3849e5867f5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/6b39f37f5f3644d1bf59c1af70ccf8c2] 2024-11-13T18:24:40,304 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:40,304 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/B is initiating minor compaction (all files) 2024-11-13T18:24:40,304 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/B in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:40,304 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/b5d761ee33e64577a2b99308c26b7d70, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/45f73acbb71144929af6fd1b3eb12d00, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/bfc597fd28c94a4fa957aae7e5877669] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=36.3 K 2024-11-13T18:24:40,311 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11bce6e5d5984346a58235e47e0427f9, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731522274800 2024-11-13T18:24:40,320 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d761ee33e64577a2b99308c26b7d70, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731522274800 2024-11-13T18:24:40,323 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef5c69d62d8f457a8824f3849e5867f5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731522276313 2024-11-13T18:24:40,324 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 45f73acbb71144929af6fd1b3eb12d00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731522276313 2024-11-13T18:24:40,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:24:40,324 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b39f37f5f3644d1bf59c1af70ccf8c2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731522278726 2024-11-13T18:24:40,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:40,325 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:24:40,325 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc597fd28c94a4fa957aae7e5877669, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731522278726 2024-11-13T18:24:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:40,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a4c7fc69b6cb416794fe4863cc75d7b5_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522279491/Put/seqid=0 2024-11-13T18:24:40,402 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:40,403 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/08b029c6bc1c4a19b859675d761e2ff1 is 50, key is test_row_0/B:col10/1731522278747/Put/seqid=0 2024-11-13T18:24:40,407 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:40,427 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111343bba4ee377a4162b6a99b8f53a86f50_a3e5c2fc3660c99b882b8a9ec24a558f store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:40,430 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111343bba4ee377a4162b6a99b8f53a86f50_a3e5c2fc3660c99b882b8a9ec24a558f, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:40,430 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111343bba4ee377a4162b6a99b8f53a86f50_a3e5c2fc3660c99b882b8a9ec24a558f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:40,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742012_1188 (size=12454) 2024-11-13T18:24:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:24:40,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742013_1189 (size=12949) 2024-11-13T18:24:40,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742014_1190 (size=4469) 2024-11-13T18:24:40,506 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#A#compaction#155 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:40,507 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/2760a408e47d49d08cccf6dad872ee18 is 175, key is test_row_0/A:col10/1731522278747/Put/seqid=0 2024-11-13T18:24:40,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742015_1191 (size=31903) 2024-11-13T18:24:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:40,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:40,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:40,885 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a4c7fc69b6cb416794fe4863cc75d7b5_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4c7fc69b6cb416794fe4863cc75d7b5_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:40,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/bc284543b15b43d596196fb4da32383f, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:40,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/bc284543b15b43d596196fb4da32383f is 175, key is test_row_0/A:col10/1731522279491/Put/seqid=0 2024-11-13T18:24:40,913 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/08b029c6bc1c4a19b859675d761e2ff1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/08b029c6bc1c4a19b859675d761e2ff1 2024-11-13T18:24:40,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742016_1192 (size=31255) 2024-11-13T18:24:40,927 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/B of a3e5c2fc3660c99b882b8a9ec24a558f into 08b029c6bc1c4a19b859675d761e2ff1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:40,927 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:40,927 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/B, priority=13, startTime=1731522280287; duration=0sec 2024-11-13T18:24:40,929 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:40,929 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:B 2024-11-13T18:24:40,929 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/bc284543b15b43d596196fb4da32383f 2024-11-13T18:24:40,929 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:40,935 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:40,935 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/C is initiating minor compaction (all files) 2024-11-13T18:24:40,935 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/C in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:40,936 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/67010b0e81ff47ba8d94fe237c11f2e1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/df5e2fbe4ab24af9b2a44eef3d258d75, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cde1eacd1105472e9f51279aea2b12b0] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=36.3 K 2024-11-13T18:24:40,937 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 67010b0e81ff47ba8d94fe237c11f2e1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731522274800 2024-11-13T18:24:40,942 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting df5e2fbe4ab24af9b2a44eef3d258d75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731522276313 2024-11-13T18:24:40,944 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting cde1eacd1105472e9f51279aea2b12b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731522278726 2024-11-13T18:24:40,948 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/2760a408e47d49d08cccf6dad872ee18 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/2760a408e47d49d08cccf6dad872ee18 2024-11-13T18:24:40,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/eb3f4b1b04124d08afbbc3f6f759d51b is 50, key is test_row_0/B:col10/1731522279491/Put/seqid=0 2024-11-13T18:24:40,972 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/A of a3e5c2fc3660c99b882b8a9ec24a558f into 2760a408e47d49d08cccf6dad872ee18(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:40,972 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:40,972 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/A, priority=13, startTime=1731522280287; duration=0sec 2024-11-13T18:24:40,972 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:40,972 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:A 2024-11-13T18:24:40,977 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#C#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:40,978 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/4af20a11b12f473387518445d5981fd5 is 50, key is test_row_0/C:col10/1731522278747/Put/seqid=0 2024-11-13T18:24:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:24:41,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742017_1193 (size=12301) 2024-11-13T18:24:41,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742018_1194 (size=12949) 2024-11-13T18:24:41,031 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/eb3f4b1b04124d08afbbc3f6f759d51b 2024-11-13T18:24:41,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522340994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:41,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37980 deadline: 1731522341026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,035 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8343 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:41,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/2eb710f87c754efd949b30769f802a75 is 50, key is test_row_0/C:col10/1731522279491/Put/seqid=0 2024-11-13T18:24:41,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742019_1195 (size=12301) 2024-11-13T18:24:41,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522341147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:41,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731522341259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:41,266 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_1' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8290 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:41,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37934 deadline: 1731522341256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:41,268 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8303 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:41,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37926 deadline: 1731522341305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:41,314 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in a3e5c2fc3660c99b882b8a9ec24a558f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8337 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:24:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:41,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522341365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:41,456 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/4af20a11b12f473387518445d5981fd5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4af20a11b12f473387518445d5981fd5 2024-11-13T18:24:41,483 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/C of a3e5c2fc3660c99b882b8a9ec24a558f into 4af20a11b12f473387518445d5981fd5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:41,483 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:41,483 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/C, priority=13, startTime=1731522280288; duration=0sec 2024-11-13T18:24:41,483 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:41,484 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:C 2024-11-13T18:24:41,541 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/2eb710f87c754efd949b30769f802a75 2024-11-13T18:24:41,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/bc284543b15b43d596196fb4da32383f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bc284543b15b43d596196fb4da32383f 2024-11-13T18:24:41,598 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bc284543b15b43d596196fb4da32383f, entries=150, sequenceid=276, filesize=30.5 K 2024-11-13T18:24:41,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/eb3f4b1b04124d08afbbc3f6f759d51b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eb3f4b1b04124d08afbbc3f6f759d51b 2024-11-13T18:24:41,616 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eb3f4b1b04124d08afbbc3f6f759d51b, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T18:24:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/2eb710f87c754efd949b30769f802a75 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/2eb710f87c754efd949b30769f802a75 2024-11-13T18:24:41,630 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/2eb710f87c754efd949b30769f802a75, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T18:24:41,632 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a3e5c2fc3660c99b882b8a9ec24a558f in 1307ms, sequenceid=276, compaction requested=false 2024-11-13T18:24:41,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:41,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:41,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-13T18:24:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-13T18:24:41,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-13T18:24:41,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7930 sec 2024-11-13T18:24:41,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8060 sec 2024-11-13T18:24:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:41,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:24:41,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:41,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139a70d705e46a4ec295abb11d2657f771_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522280909/Put/seqid=0 2024-11-13T18:24:41,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742020_1196 (size=14994) 2024-11-13T18:24:41,807 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:41,839 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139a70d705e46a4ec295abb11d2657f771_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139a70d705e46a4ec295abb11d2657f771_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:41,843 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/788d4a676f4e4f18bd9fac77fffc75d6, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:41,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/788d4a676f4e4f18bd9fac77fffc75d6 is 175, key is test_row_0/A:col10/1731522280909/Put/seqid=0 2024-11-13T18:24:41,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522341846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:41,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742021_1197 (size=39949) 2024-11-13T18:24:41,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/788d4a676f4e4f18bd9fac77fffc75d6 2024-11-13T18:24:41,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522341966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:41,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/eaf2f5873a5d43ac9321ae987a5bbfa3 is 50, key is test_row_0/B:col10/1731522280909/Put/seqid=0 2024-11-13T18:24:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:24:42,003 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:42,004 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-13T18:24:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:24:42,012 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:42,013 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:42,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:42,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742022_1198 (size=12301) 2024-11-13T18:24:42,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/eaf2f5873a5d43ac9321ae987a5bbfa3 2024-11-13T18:24:42,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/e378ffccdfd742539b6fd6dba5929611 is 50, key is test_row_0/C:col10/1731522280909/Put/seqid=0 2024-11-13T18:24:42,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742023_1199 (size=12301) 2024-11-13T18:24:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:24:42,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/e378ffccdfd742539b6fd6dba5929611 2024-11-13T18:24:42,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/788d4a676f4e4f18bd9fac77fffc75d6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/788d4a676f4e4f18bd9fac77fffc75d6 2024-11-13T18:24:42,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T18:24:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:42,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:42,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/788d4a676f4e4f18bd9fac77fffc75d6, entries=200, sequenceid=299, filesize=39.0 K 2024-11-13T18:24:42,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522342187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:42,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/eaf2f5873a5d43ac9321ae987a5bbfa3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eaf2f5873a5d43ac9321ae987a5bbfa3 2024-11-13T18:24:42,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eaf2f5873a5d43ac9321ae987a5bbfa3, entries=150, sequenceid=299, filesize=12.0 K 2024-11-13T18:24:42,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/e378ffccdfd742539b6fd6dba5929611 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e378ffccdfd742539b6fd6dba5929611 2024-11-13T18:24:42,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e378ffccdfd742539b6fd6dba5929611, entries=150, sequenceid=299, filesize=12.0 K 2024-11-13T18:24:42,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a3e5c2fc3660c99b882b8a9ec24a558f in 596ms, sequenceid=299, compaction requested=true 2024-11-13T18:24:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:42,287 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:42,288 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:42,288 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/A is initiating minor compaction (all files) 2024-11-13T18:24:42,289 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/A in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:42,289 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/2760a408e47d49d08cccf6dad872ee18, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bc284543b15b43d596196fb4da32383f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/788d4a676f4e4f18bd9fac77fffc75d6] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=100.7 K 2024-11-13T18:24:42,289 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:42,289 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/2760a408e47d49d08cccf6dad872ee18, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bc284543b15b43d596196fb4da32383f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/788d4a676f4e4f18bd9fac77fffc75d6] 2024-11-13T18:24:42,290 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2760a408e47d49d08cccf6dad872ee18, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731522278726 2024-11-13T18:24:42,290 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc284543b15b43d596196fb4da32383f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731522279491 2024-11-13T18:24:42,291 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 788d4a676f4e4f18bd9fac77fffc75d6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731522280909 2024-11-13T18:24:42,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:42,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:42,301 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:42,302 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:42,302 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/B is initiating minor compaction (all files) 2024-11-13T18:24:42,302 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/B in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:42,303 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/08b029c6bc1c4a19b859675d761e2ff1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eb3f4b1b04124d08afbbc3f6f759d51b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eaf2f5873a5d43ac9321ae987a5bbfa3] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=36.7 K 2024-11-13T18:24:42,303 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b029c6bc1c4a19b859675d761e2ff1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731522278726 2024-11-13T18:24:42,303 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:42,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:42,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:42,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:42,308 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting eb3f4b1b04124d08afbbc3f6f759d51b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731522279491 2024-11-13T18:24:42,309 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting eaf2f5873a5d43ac9321ae987a5bbfa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731522280909 2024-11-13T18:24:42,325 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111353a20a9f36a04e708f553f75361a198c_a3e5c2fc3660c99b882b8a9ec24a558f store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:24:42,328 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111353a20a9f36a04e708f553f75361a198c_a3e5c2fc3660c99b882b8a9ec24a558f, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:42,328 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111353a20a9f36a04e708f553f75361a198c_a3e5c2fc3660c99b882b8a9ec24a558f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:42,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T18:24:42,341 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:42,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:42,342 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:24:42,342 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/421d49aeb67045a1b9e84b21dd6a6329 is 50, key is test_row_0/B:col10/1731522280909/Put/seqid=0 2024-11-13T18:24:42,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:42,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:42,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:42,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:42,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:42,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:42,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111362f42db0a055423a8cfdbb76ccfb7ae8_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522281838/Put/seqid=0 2024-11-13T18:24:42,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742024_1200 (size=4469) 2024-11-13T18:24:42,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742025_1201 (size=13051) 2024-11-13T18:24:42,425 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/421d49aeb67045a1b9e84b21dd6a6329 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/421d49aeb67045a1b9e84b21dd6a6329 2024-11-13T18:24:42,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742026_1202 (size=12454) 2024-11-13T18:24:42,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:42,443 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/B of a3e5c2fc3660c99b882b8a9ec24a558f into 421d49aeb67045a1b9e84b21dd6a6329(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:42,443 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:42,443 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/B, priority=13, startTime=1731522282300; duration=0sec 2024-11-13T18:24:42,443 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:42,443 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:B 2024-11-13T18:24:42,443 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:42,453 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:42,454 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/C is initiating minor compaction (all files) 2024-11-13T18:24:42,454 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/C in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:42,454 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4af20a11b12f473387518445d5981fd5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/2eb710f87c754efd949b30769f802a75, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e378ffccdfd742539b6fd6dba5929611] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=36.7 K 2024-11-13T18:24:42,455 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 4af20a11b12f473387518445d5981fd5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731522278726 2024-11-13T18:24:42,455 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eb710f87c754efd949b30769f802a75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731522279491 2024-11-13T18:24:42,456 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e378ffccdfd742539b6fd6dba5929611, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731522280909 2024-11-13T18:24:42,456 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111362f42db0a055423a8cfdbb76ccfb7ae8_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111362f42db0a055423a8cfdbb76ccfb7ae8_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:42,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/84dfbbb26bcc4f198236411651e16a9d, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:42,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/84dfbbb26bcc4f198236411651e16a9d is 175, key is test_row_0/A:col10/1731522281838/Put/seqid=0 2024-11-13T18:24:42,501 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:42,502 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/c84b64aefef74a84b9e5f614ff5a598f is 50, key is test_row_0/C:col10/1731522280909/Put/seqid=0 2024-11-13T18:24:42,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742027_1203 (size=31255) 2024-11-13T18:24:42,507 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/84dfbbb26bcc4f198236411651e16a9d 2024-11-13T18:24:42,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:42,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742028_1204 (size=13051) 2024-11-13T18:24:42,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/29cfebdbb5b44e65b354a82ddfd528e4 is 50, key is test_row_0/B:col10/1731522281838/Put/seqid=0 2024-11-13T18:24:42,630 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/c84b64aefef74a84b9e5f614ff5a598f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/c84b64aefef74a84b9e5f614ff5a598f 2024-11-13T18:24:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:24:42,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742029_1205 (size=12301) 2024-11-13T18:24:42,669 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/C of a3e5c2fc3660c99b882b8a9ec24a558f into c84b64aefef74a84b9e5f614ff5a598f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:42,670 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:42,670 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/C, priority=13, startTime=1731522282304; duration=0sec 2024-11-13T18:24:42,670 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:42,670 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:C 2024-11-13T18:24:42,803 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#A#compaction#162 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:42,804 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/a093e656f67e43d2952c03812a3ee002 is 175, key is test_row_0/A:col10/1731522280909/Put/seqid=0 2024-11-13T18:24:42,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742030_1206 (size=32005) 2024-11-13T18:24:42,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522342896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:43,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522343031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:43,067 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/29cfebdbb5b44e65b354a82ddfd528e4 2024-11-13T18:24:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/af5ef1475d86461bbfaebd3e26ffb0f5 is 50, key is test_row_0/C:col10/1731522281838/Put/seqid=0 2024-11-13T18:24:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:24:43,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742031_1207 (size=12301) 2024-11-13T18:24:43,213 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/af5ef1475d86461bbfaebd3e26ffb0f5 2024-11-13T18:24:43,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/84dfbbb26bcc4f198236411651e16a9d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/84dfbbb26bcc4f198236411651e16a9d 2024-11-13T18:24:43,244 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/84dfbbb26bcc4f198236411651e16a9d, entries=150, sequenceid=311, filesize=30.5 K 2024-11-13T18:24:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/29cfebdbb5b44e65b354a82ddfd528e4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/29cfebdbb5b44e65b354a82ddfd528e4 2024-11-13T18:24:43,261 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/29cfebdbb5b44e65b354a82ddfd528e4, entries=150, sequenceid=311, filesize=12.0 K 2024-11-13T18:24:43,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/af5ef1475d86461bbfaebd3e26ffb0f5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/af5ef1475d86461bbfaebd3e26ffb0f5 2024-11-13T18:24:43,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522343245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:43,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:43,299 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/af5ef1475d86461bbfaebd3e26ffb0f5, entries=150, sequenceid=311, filesize=12.0 K 2024-11-13T18:24:43,300 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/a093e656f67e43d2952c03812a3ee002 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a093e656f67e43d2952c03812a3ee002 2024-11-13T18:24:43,307 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a3e5c2fc3660c99b882b8a9ec24a558f in 966ms, sequenceid=311, compaction requested=false 2024-11-13T18:24:43,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:43,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:43,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-13T18:24:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-13T18:24:43,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T18:24:43,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3060 sec 2024-11-13T18:24:43,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.3240 sec 2024-11-13T18:24:43,353 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/A of a3e5c2fc3660c99b882b8a9ec24a558f into a093e656f67e43d2952c03812a3ee002(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-13T18:24:43,353 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:43,353 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/A, priority=13, startTime=1731522282287; duration=1sec 2024-11-13T18:24:43,353 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:43,353 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:A 2024-11-13T18:24:43,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522282750, max timestamp=1731522283587Num uniques 859; 2024-11-13T18:24:43,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:43,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-13T18:24:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:24:43,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134c0be8c838434a9daea306c98c2a720e_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522282819/Put/seqid=0 2024-11-13T18:24:43,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742032_1208 (size=12454) 2024-11-13T18:24:43,733 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:43,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522343778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:43,791 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134c0be8c838434a9daea306c98c2a720e_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134c0be8c838434a9daea306c98c2a720e_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:43,810 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/a418432a87b542068883dccb483d0de1, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:43,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/a418432a87b542068883dccb483d0de1 is 175, key is test_row_0/A:col10/1731522282819/Put/seqid=0 2024-11-13T18:24:43,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742033_1209 (size=31255) 2024-11-13T18:24:43,863 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/a418432a87b542068883dccb483d0de1 2024-11-13T18:24:43,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522343897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:43,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/1d9187faca4341548b3b23f60b07a3d6 is 50, key is test_row_0/B:col10/1731522282819/Put/seqid=0 2024-11-13T18:24:43,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742034_1210 (size=12301) 2024-11-13T18:24:44,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/1d9187faca4341548b3b23f60b07a3d6 2024-11-13T18:24:44,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/5d17d881f2244ffa936f28d3275425c2 is 50, key is test_row_0/C:col10/1731522282819/Put/seqid=0 2024-11-13T18:24:44,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742035_1211 (size=12301) 2024-11-13T18:24:44,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522344109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:24:44,168 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:44,179 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-13T18:24:44,181 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:44,182 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:44,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:24:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:24:44,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T18:24:44,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:44,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:44,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:44,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:44,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522344425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:24:44,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T18:24:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:44,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:44,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/5d17d881f2244ffa936f28d3275425c2 2024-11-13T18:24:44,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/a418432a87b542068883dccb483d0de1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a418432a87b542068883dccb483d0de1 2024-11-13T18:24:44,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a418432a87b542068883dccb483d0de1, entries=150, sequenceid=336, filesize=30.5 K 2024-11-13T18:24:44,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/1d9187faca4341548b3b23f60b07a3d6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1d9187faca4341548b3b23f60b07a3d6 2024-11-13T18:24:44,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1d9187faca4341548b3b23f60b07a3d6, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T18:24:44,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/5d17d881f2244ffa936f28d3275425c2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/5d17d881f2244ffa936f28d3275425c2 2024-11-13T18:24:44,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/5d17d881f2244ffa936f28d3275425c2, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T18:24:44,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for a3e5c2fc3660c99b882b8a9ec24a558f in 958ms, sequenceid=336, compaction requested=true 2024-11-13T18:24:44,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:44,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:44,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:44,557 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:44,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:44,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:44,557 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:44,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:44,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:44,559 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:44,560 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/B is initiating minor compaction (all files) 2024-11-13T18:24:44,560 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/B in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:44,560 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/421d49aeb67045a1b9e84b21dd6a6329, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/29cfebdbb5b44e65b354a82ddfd528e4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1d9187faca4341548b3b23f60b07a3d6] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=36.8 K 2024-11-13T18:24:44,560 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:44,560 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/A is initiating minor compaction (all files) 2024-11-13T18:24:44,560 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/A in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:44,560 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a093e656f67e43d2952c03812a3ee002, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/84dfbbb26bcc4f198236411651e16a9d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a418432a87b542068883dccb483d0de1] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=92.3 K 2024-11-13T18:24:44,560 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:44,560 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a093e656f67e43d2952c03812a3ee002, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/84dfbbb26bcc4f198236411651e16a9d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a418432a87b542068883dccb483d0de1] 2024-11-13T18:24:44,563 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting a093e656f67e43d2952c03812a3ee002, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731522280909 2024-11-13T18:24:44,563 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 421d49aeb67045a1b9e84b21dd6a6329, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731522280909 2024-11-13T18:24:44,564 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84dfbbb26bcc4f198236411651e16a9d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731522281798 2024-11-13T18:24:44,564 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 29cfebdbb5b44e65b354a82ddfd528e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731522281798 2024-11-13T18:24:44,566 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting a418432a87b542068883dccb483d0de1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731522282819 2024-11-13T18:24:44,567 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d9187faca4341548b3b23f60b07a3d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731522282819 2024-11-13T18:24:44,586 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:44,596 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#B#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:44,597 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/7d5c54b3b3e04ef09f8a8da091ea4a56 is 50, key is test_row_0/B:col10/1731522282819/Put/seqid=0 2024-11-13T18:24:44,605 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411139d9461ddd31b4c2baa3a70cd575d0b2e_a3e5c2fc3660c99b882b8a9ec24a558f store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:44,608 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411139d9461ddd31b4c2baa3a70cd575d0b2e_a3e5c2fc3660c99b882b8a9ec24a558f, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:44,608 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d9461ddd31b4c2baa3a70cd575d0b2e_a3e5c2fc3660c99b882b8a9ec24a558f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:44,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742036_1212 (size=13153) 2024-11-13T18:24:44,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T18:24:44,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:44,661 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:24:44,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:44,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:44,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:44,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:44,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:44,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:44,668 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/7d5c54b3b3e04ef09f8a8da091ea4a56 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/7d5c54b3b3e04ef09f8a8da091ea4a56 2024-11-13T18:24:44,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742037_1213 (size=4469) 2024-11-13T18:24:44,677 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#A#compaction#171 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:44,678 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/16ddcb97959a488488fb674ee0855d41 is 175, key is test_row_0/A:col10/1731522282819/Put/seqid=0 2024-11-13T18:24:44,686 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/B of a3e5c2fc3660c99b882b8a9ec24a558f into 7d5c54b3b3e04ef09f8a8da091ea4a56(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:44,686 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:44,686 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/B, priority=13, startTime=1731522284557; duration=0sec 2024-11-13T18:24:44,686 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:44,686 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:B 2024-11-13T18:24:44,686 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:44,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111387c2c3c5a3a4487ea242ed9461b50eba_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522283743/Put/seqid=0 2024-11-13T18:24:44,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742038_1214 (size=32107) 2024-11-13T18:24:44,704 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:44,704 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/C is initiating minor compaction (all files) 2024-11-13T18:24:44,704 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/C in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:44,704 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/c84b64aefef74a84b9e5f614ff5a598f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/af5ef1475d86461bbfaebd3e26ffb0f5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/5d17d881f2244ffa936f28d3275425c2] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=36.8 K 2024-11-13T18:24:44,711 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c84b64aefef74a84b9e5f614ff5a598f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731522280909 2024-11-13T18:24:44,723 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting af5ef1475d86461bbfaebd3e26ffb0f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731522281798 2024-11-13T18:24:44,731 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d17d881f2244ffa936f28d3275425c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731522282819 2024-11-13T18:24:44,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742039_1215 (size=12454) 2024-11-13T18:24:44,755 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#C#compaction#174 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:44,755 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/62725bbc8efd4e2eada8dade3c49889b is 50, key is test_row_0/C:col10/1731522282819/Put/seqid=0 2024-11-13T18:24:44,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742040_1216 (size=13153) 2024-11-13T18:24:44,780 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/62725bbc8efd4e2eada8dade3c49889b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62725bbc8efd4e2eada8dade3c49889b 2024-11-13T18:24:44,801 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/C of a3e5c2fc3660c99b882b8a9ec24a558f into 62725bbc8efd4e2eada8dade3c49889b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:44,801 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:44,801 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/C, priority=13, startTime=1731522284557; duration=0sec 2024-11-13T18:24:44,801 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:44,801 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:C 2024-11-13T18:24:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:24:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:44,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. as already flushing 2024-11-13T18:24:45,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522345113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:45,126 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/16ddcb97959a488488fb674ee0855d41 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/16ddcb97959a488488fb674ee0855d41 2024-11-13T18:24:45,134 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/A of a3e5c2fc3660c99b882b8a9ec24a558f into 16ddcb97959a488488fb674ee0855d41(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:45,134 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:45,134 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/A, priority=13, startTime=1731522284557; duration=0sec 2024-11-13T18:24:45,134 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:45,134 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:A 2024-11-13T18:24:45,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:45,152 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111387c2c3c5a3a4487ea242ed9461b50eba_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111387c2c3c5a3a4487ea242ed9461b50eba_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:45,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/9fd94b6bcdae4fc0b89f1f2a883853d1, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:45,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/9fd94b6bcdae4fc0b89f1f2a883853d1 is 175, key is test_row_0/A:col10/1731522283743/Put/seqid=0 2024-11-13T18:24:45,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742041_1217 (size=31255) 2024-11-13T18:24:45,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522345244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:24:45,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522345459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:45,638 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/9fd94b6bcdae4fc0b89f1f2a883853d1 2024-11-13T18:24:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/453ae57b09ff4f168500c070f7a66944 is 50, key is test_row_0/B:col10/1731522283743/Put/seqid=0 2024-11-13T18:24:45,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742042_1218 (size=12301) 2024-11-13T18:24:45,759 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/453ae57b09ff4f168500c070f7a66944 2024-11-13T18:24:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/4bbb569821904077a708bd22872b8ac3 is 50, key is test_row_0/C:col10/1731522283743/Put/seqid=0 2024-11-13T18:24:45,784 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-11-13T18:24:45,784 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:45,784 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:45,784 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:45,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:45,785 INFO [Thread-596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-596. 2024-11-13T18:24:45,785 DEBUG [Thread-596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:45,785 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:45,786 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:45,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:45,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37964 deadline: 1731522345778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:24:45,795 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-11-13T18:24:45,795 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:45,795 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:45,795 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:45,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:45,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742043_1219 (size=12301) 2024-11-13T18:24:45,812 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/4bbb569821904077a708bd22872b8ac3 2024-11-13T18:24:45,813 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-11-13T18:24:45,813 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:45,813 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:45,813 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:45,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:45,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/9fd94b6bcdae4fc0b89f1f2a883853d1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9fd94b6bcdae4fc0b89f1f2a883853d1 2024-11-13T18:24:45,866 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9fd94b6bcdae4fc0b89f1f2a883853d1, entries=150, sequenceid=351, filesize=30.5 K 2024-11-13T18:24:45,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/453ae57b09ff4f168500c070f7a66944 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/453ae57b09ff4f168500c070f7a66944 2024-11-13T18:24:45,898 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/453ae57b09ff4f168500c070f7a66944, entries=150, sequenceid=351, filesize=12.0 K 2024-11-13T18:24:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/4bbb569821904077a708bd22872b8ac3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4bbb569821904077a708bd22872b8ac3 2024-11-13T18:24:45,929 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4bbb569821904077a708bd22872b8ac3, entries=150, sequenceid=351, filesize=12.0 K 2024-11-13T18:24:45,935 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a3e5c2fc3660c99b882b8a9ec24a558f in 1274ms, sequenceid=351, compaction requested=false 2024-11-13T18:24:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-13T18:24:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-13T18:24:45,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-13T18:24:45,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7560 sec 2024-11-13T18:24:45,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.7800 sec 2024-11-13T18:24:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:46,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:24:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:46,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:46,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:46,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:46,306 INFO [Thread-581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-581. 2024-11-13T18:24:46,306 DEBUG [Thread-581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:46,306 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:46,306 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:46,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:24:46,330 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:46,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137acc58ae0b7c4b80a3a8f2f6e413ece4_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522285108/Put/seqid=0 2024-11-13T18:24:46,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742044_1220 (size=12454) 2024-11-13T18:24:46,794 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:46,821 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137acc58ae0b7c4b80a3a8f2f6e413ece4_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137acc58ae0b7c4b80a3a8f2f6e413ece4_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:46,826 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/3abad75db26448a5b869cc2215b4529a, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:46,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/3abad75db26448a5b869cc2215b4529a is 175, key is test_row_0/A:col10/1731522285108/Put/seqid=0 2024-11-13T18:24:46,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742045_1221 (size=31255) 2024-11-13T18:24:47,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/3abad75db26448a5b869cc2215b4529a 2024-11-13T18:24:47,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/f27e803f6b224adcb1b3aa5ef008f536 is 50, key is test_row_0/B:col10/1731522285108/Put/seqid=0 2024-11-13T18:24:47,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742046_1222 (size=12301) 2024-11-13T18:24:47,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/f27e803f6b224adcb1b3aa5ef008f536 2024-11-13T18:24:47,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/0238344a7c7e4375a8fe1ac564f3c133 is 50, key is test_row_0/C:col10/1731522285108/Put/seqid=0 2024-11-13T18:24:47,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742047_1223 (size=12301) 2024-11-13T18:24:47,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/0238344a7c7e4375a8fe1ac564f3c133 2024-11-13T18:24:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/3abad75db26448a5b869cc2215b4529a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/3abad75db26448a5b869cc2215b4529a 2024-11-13T18:24:47,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/3abad75db26448a5b869cc2215b4529a, entries=150, sequenceid=372, filesize=30.5 K 2024-11-13T18:24:47,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/f27e803f6b224adcb1b3aa5ef008f536 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f27e803f6b224adcb1b3aa5ef008f536 2024-11-13T18:24:47,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f27e803f6b224adcb1b3aa5ef008f536, entries=150, sequenceid=372, filesize=12.0 K 2024-11-13T18:24:47,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/0238344a7c7e4375a8fe1ac564f3c133 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0238344a7c7e4375a8fe1ac564f3c133 2024-11-13T18:24:47,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0238344a7c7e4375a8fe1ac564f3c133, entries=150, sequenceid=372, filesize=12.0 K 2024-11-13T18:24:47,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for a3e5c2fc3660c99b882b8a9ec24a558f in 1458ms, sequenceid=372, compaction requested=true 2024-11-13T18:24:47,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:47,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:24:47,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:47,762 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:47,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:24:47,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3e5c2fc3660c99b882b8a9ec24a558f:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:24:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:47,767 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:47,769 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:47,769 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/B is initiating minor compaction (all files) 2024-11-13T18:24:47,769 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/B in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:47,769 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/7d5c54b3b3e04ef09f8a8da091ea4a56, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/453ae57b09ff4f168500c070f7a66944, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f27e803f6b224adcb1b3aa5ef008f536] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=36.9 K 2024-11-13T18:24:47,771 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d5c54b3b3e04ef09f8a8da091ea4a56, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731522282819 2024-11-13T18:24:47,779 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 453ae57b09ff4f168500c070f7a66944, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731522283743 2024-11-13T18:24:47,780 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:47,780 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/A is initiating minor compaction (all files) 2024-11-13T18:24:47,780 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/A in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:47,780 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/16ddcb97959a488488fb674ee0855d41, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9fd94b6bcdae4fc0b89f1f2a883853d1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/3abad75db26448a5b869cc2215b4529a] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=92.4 K 2024-11-13T18:24:47,780 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:47,780 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/16ddcb97959a488488fb674ee0855d41, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9fd94b6bcdae4fc0b89f1f2a883853d1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/3abad75db26448a5b869cc2215b4529a] 2024-11-13T18:24:47,781 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting f27e803f6b224adcb1b3aa5ef008f536, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731522285089 2024-11-13T18:24:47,781 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16ddcb97959a488488fb674ee0855d41, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731522282819 2024-11-13T18:24:47,782 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd94b6bcdae4fc0b89f1f2a883853d1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731522283743 2024-11-13T18:24:47,782 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3abad75db26448a5b869cc2215b4529a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731522285089 2024-11-13T18:24:47,814 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#B#compaction#180 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:47,815 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/9b6cd0dabb034346b9bea297244a49b9 is 50, key is test_row_0/B:col10/1731522285108/Put/seqid=0 2024-11-13T18:24:47,834 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:47,857 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ba9a4564a82f481e80a50a8f341e542d_a3e5c2fc3660c99b882b8a9ec24a558f store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:47,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742048_1224 (size=13255) 2024-11-13T18:24:47,904 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ba9a4564a82f481e80a50a8f341e542d_a3e5c2fc3660c99b882b8a9ec24a558f, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:47,904 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ba9a4564a82f481e80a50a8f341e542d_a3e5c2fc3660c99b882b8a9ec24a558f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:47,951 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/9b6cd0dabb034346b9bea297244a49b9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9b6cd0dabb034346b9bea297244a49b9 2024-11-13T18:24:47,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742049_1225 (size=4469) 2024-11-13T18:24:47,992 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/B of a3e5c2fc3660c99b882b8a9ec24a558f into 9b6cd0dabb034346b9bea297244a49b9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:47,992 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:47,992 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/B, priority=13, startTime=1731522287762; duration=0sec 2024-11-13T18:24:47,992 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:24:47,992 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:B 2024-11-13T18:24:47,992 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:24:47,998 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:24:47,998 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): a3e5c2fc3660c99b882b8a9ec24a558f/C is initiating minor compaction (all files) 2024-11-13T18:24:47,999 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3e5c2fc3660c99b882b8a9ec24a558f/C in TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:47,999 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62725bbc8efd4e2eada8dade3c49889b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4bbb569821904077a708bd22872b8ac3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0238344a7c7e4375a8fe1ac564f3c133] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp, totalSize=36.9 K 2024-11-13T18:24:47,999 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 62725bbc8efd4e2eada8dade3c49889b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731522282819 2024-11-13T18:24:48,000 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bbb569821904077a708bd22872b8ac3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731522283743 2024-11-13T18:24:48,000 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 0238344a7c7e4375a8fe1ac564f3c133, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731522285089 2024-11-13T18:24:48,031 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#C#compaction#182 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:48,032 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/826c547d64a74edf94a00efdefe29630 is 50, key is test_row_0/C:col10/1731522285108/Put/seqid=0 2024-11-13T18:24:48,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742050_1226 (size=13255) 2024-11-13T18:24:48,146 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/826c547d64a74edf94a00efdefe29630 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/826c547d64a74edf94a00efdefe29630 2024-11-13T18:24:48,186 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/C of a3e5c2fc3660c99b882b8a9ec24a558f into 826c547d64a74edf94a00efdefe29630(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:24:48,187 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:48,187 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/C, priority=13, startTime=1731522287763; duration=0sec 2024-11-13T18:24:48,187 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:48,187 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:C 2024-11-13T18:24:48,480 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3e5c2fc3660c99b882b8a9ec24a558f#A#compaction#181 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:24:48,481 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/14ea5d7bd67a488585a46d3a1c91e2bd is 175, key is test_row_0/A:col10/1731522285108/Put/seqid=0 2024-11-13T18:24:48,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742051_1227 (size=32209) 2024-11-13T18:24:48,975 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/14ea5d7bd67a488585a46d3a1c91e2bd as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/14ea5d7bd67a488585a46d3a1c91e2bd 2024-11-13T18:24:49,014 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3e5c2fc3660c99b882b8a9ec24a558f/A of a3e5c2fc3660c99b882b8a9ec24a558f into 14ea5d7bd67a488585a46d3a1c91e2bd(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-13T18:24:49,014 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3e5c2fc3660c99b882b8a9ec24a558f: 2024-11-13T18:24:49,014 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., storeName=a3e5c2fc3660c99b882b8a9ec24a558f/A, priority=13, startTime=1731522287762; duration=1sec 2024-11-13T18:24:49,014 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:24:49,014 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3e5c2fc3660c99b882b8a9ec24a558f:A 2024-11-13T18:24:51,105 INFO [Thread-579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-579. 2024-11-13T18:24:51,105 DEBUG [Thread-579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:51,105 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:51,105 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:51,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:51,302 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-13T18:24:51,302 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:51,302 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:51,302 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:51,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:51,336 INFO [Thread-585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-585. 2024-11-13T18:24:51,336 DEBUG [Thread-585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:51,336 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:51,336 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:51,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:51,338 INFO [Thread-583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-583. 2024-11-13T18:24:51,338 DEBUG [Thread-583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:24:51,338 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:51,338 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-13T18:24:51,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1118 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1055 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 531 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1590 rows 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 521 2024-11-13T18:24:51,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1561 rows 2024-11-13T18:24:51,338 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:24:51,338 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:24:51,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:51,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:51,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:24:51,340 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:24:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T18:24:51,352 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522291352"}]},"ts":"1731522291352"} 2024-11-13T18:24:51,370 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:24:51,370 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:24:51,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:24:51,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, UNASSIGN}] 2024-11-13T18:24:51,375 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, UNASSIGN 2024-11-13T18:24:51,376 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=a3e5c2fc3660c99b882b8a9ec24a558f, regionState=CLOSING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:24:51,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, UNASSIGN because future has completed 2024-11-13T18:24:51,384 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:24:51,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:24:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T18:24:51,540 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:51,540 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:24:51,540 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing a3e5c2fc3660c99b882b8a9ec24a558f, disabling compactions & flushes 2024-11-13T18:24:51,540 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:51,540 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:51,540 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. after waiting 0 ms 2024-11-13T18:24:51,540 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:51,541 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing a3e5c2fc3660c99b882b8a9ec24a558f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T18:24:51,541 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=A 2024-11-13T18:24:51,541 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:51,541 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=B 2024-11-13T18:24:51,541 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:51,541 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3e5c2fc3660c99b882b8a9ec24a558f, store=C 2024-11-13T18:24:51,541 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:51,573 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b1014bb4b63d497cbd22629e3f2dec84_a3e5c2fc3660c99b882b8a9ec24a558f is 50, key is test_row_0/A:col10/1731522291331/Put/seqid=0 2024-11-13T18:24:51,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742052_1228 (size=12454) 2024-11-13T18:24:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T18:24:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T18:24:52,024 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:52,062 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b1014bb4b63d497cbd22629e3f2dec84_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1014bb4b63d497cbd22629e3f2dec84_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:52,069 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/bf3d6246fce444668fa4385dc209998d, store: [table=TestAcidGuarantees family=A region=a3e5c2fc3660c99b882b8a9ec24a558f] 2024-11-13T18:24:52,070 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/bf3d6246fce444668fa4385dc209998d is 175, key is test_row_0/A:col10/1731522291331/Put/seqid=0 2024-11-13T18:24:52,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742053_1229 (size=31255) 2024-11-13T18:24:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T18:24:52,521 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/bf3d6246fce444668fa4385dc209998d 2024-11-13T18:24:52,549 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/fd7b5ac105b84297b678c05dcc3c4f5e is 50, key is test_row_0/B:col10/1731522291331/Put/seqid=0 2024-11-13T18:24:52,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742054_1230 (size=12301) 2024-11-13T18:24:52,609 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/fd7b5ac105b84297b678c05dcc3c4f5e 2024-11-13T18:24:52,673 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/34a75a0aefb2443fa8e88aef2171a90b is 50, key is test_row_0/C:col10/1731522291331/Put/seqid=0 2024-11-13T18:24:52,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742055_1231 (size=12301) 2024-11-13T18:24:53,167 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/34a75a0aefb2443fa8e88aef2171a90b 2024-11-13T18:24:53,203 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/A/bf3d6246fce444668fa4385dc209998d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bf3d6246fce444668fa4385dc209998d 2024-11-13T18:24:53,230 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bf3d6246fce444668fa4385dc209998d, entries=150, sequenceid=382, filesize=30.5 K 2024-11-13T18:24:53,232 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/B/fd7b5ac105b84297b678c05dcc3c4f5e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/fd7b5ac105b84297b678c05dcc3c4f5e 2024-11-13T18:24:53,249 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/fd7b5ac105b84297b678c05dcc3c4f5e, entries=150, sequenceid=382, filesize=12.0 K 2024-11-13T18:24:53,258 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/.tmp/C/34a75a0aefb2443fa8e88aef2171a90b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/34a75a0aefb2443fa8e88aef2171a90b 2024-11-13T18:24:53,311 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/34a75a0aefb2443fa8e88aef2171a90b, entries=150, sequenceid=382, filesize=12.0 K 2024-11-13T18:24:53,312 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a3e5c2fc3660c99b882b8a9ec24a558f in 1772ms, sequenceid=382, compaction requested=false 2024-11-13T18:24:53,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/b184e7ff85a8440697d74c2045b24857, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c67d1c5820d3472d84bf957e4a763557, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/86098d2ca20846a4a5725d22aa222a4e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c04bb0259ccf41d7a00130182e505a5b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d829ee2830584ee0b967de22452fedc4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/0b70fc53254e4984848bb6c836abbcb7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/38373185977a40e8aacc8c422a0ac406, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/f8b1c84e28e64f4b89ea27a1e9e923d6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/384c986889844aa9bbd18cb49803c9a5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/80b3c4fbca75431e973b310a5deb2b7c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/39aeec3a31b941cf9a2f8d43de049006, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/026582010f95475fb2371b0107e66b66, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9ee2adff2bc142adb6da460a34c4fd50, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d6e9ab6b7000425187bdac8a98bee0d8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/db21fb9913fa4d83be4e9ea6be695fa2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11550e7368b4442b883533b07483a959, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11bce6e5d5984346a58235e47e0427f9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/ef5c69d62d8f457a8824f3849e5867f5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/2760a408e47d49d08cccf6dad872ee18, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/6b39f37f5f3644d1bf59c1af70ccf8c2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bc284543b15b43d596196fb4da32383f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/788d4a676f4e4f18bd9fac77fffc75d6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a093e656f67e43d2952c03812a3ee002, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/84dfbbb26bcc4f198236411651e16a9d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/16ddcb97959a488488fb674ee0855d41, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a418432a87b542068883dccb483d0de1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9fd94b6bcdae4fc0b89f1f2a883853d1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/3abad75db26448a5b869cc2215b4529a] to archive 2024-11-13T18:24:53,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:24:53,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/b184e7ff85a8440697d74c2045b24857 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/b184e7ff85a8440697d74c2045b24857 2024-11-13T18:24:53,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c67d1c5820d3472d84bf957e4a763557 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c67d1c5820d3472d84bf957e4a763557 2024-11-13T18:24:53,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/86098d2ca20846a4a5725d22aa222a4e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/86098d2ca20846a4a5725d22aa222a4e 2024-11-13T18:24:53,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c04bb0259ccf41d7a00130182e505a5b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/c04bb0259ccf41d7a00130182e505a5b 2024-11-13T18:24:53,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d829ee2830584ee0b967de22452fedc4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d829ee2830584ee0b967de22452fedc4 2024-11-13T18:24:53,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/0b70fc53254e4984848bb6c836abbcb7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/0b70fc53254e4984848bb6c836abbcb7 2024-11-13T18:24:53,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/38373185977a40e8aacc8c422a0ac406 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/38373185977a40e8aacc8c422a0ac406 2024-11-13T18:24:53,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/f8b1c84e28e64f4b89ea27a1e9e923d6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/f8b1c84e28e64f4b89ea27a1e9e923d6 2024-11-13T18:24:53,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/384c986889844aa9bbd18cb49803c9a5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/384c986889844aa9bbd18cb49803c9a5 2024-11-13T18:24:53,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/80b3c4fbca75431e973b310a5deb2b7c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/80b3c4fbca75431e973b310a5deb2b7c 2024-11-13T18:24:53,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/39aeec3a31b941cf9a2f8d43de049006 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/39aeec3a31b941cf9a2f8d43de049006 2024-11-13T18:24:53,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/026582010f95475fb2371b0107e66b66 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/026582010f95475fb2371b0107e66b66 2024-11-13T18:24:53,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9ee2adff2bc142adb6da460a34c4fd50 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9ee2adff2bc142adb6da460a34c4fd50 2024-11-13T18:24:53,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d6e9ab6b7000425187bdac8a98bee0d8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/d6e9ab6b7000425187bdac8a98bee0d8 2024-11-13T18:24:53,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/db21fb9913fa4d83be4e9ea6be695fa2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/db21fb9913fa4d83be4e9ea6be695fa2 2024-11-13T18:24:53,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11550e7368b4442b883533b07483a959 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11550e7368b4442b883533b07483a959 2024-11-13T18:24:53,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11bce6e5d5984346a58235e47e0427f9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/11bce6e5d5984346a58235e47e0427f9 2024-11-13T18:24:53,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/ef5c69d62d8f457a8824f3849e5867f5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/ef5c69d62d8f457a8824f3849e5867f5 2024-11-13T18:24:53,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/2760a408e47d49d08cccf6dad872ee18 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/2760a408e47d49d08cccf6dad872ee18 2024-11-13T18:24:53,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/6b39f37f5f3644d1bf59c1af70ccf8c2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/6b39f37f5f3644d1bf59c1af70ccf8c2 2024-11-13T18:24:53,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bc284543b15b43d596196fb4da32383f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bc284543b15b43d596196fb4da32383f 2024-11-13T18:24:53,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/788d4a676f4e4f18bd9fac77fffc75d6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/788d4a676f4e4f18bd9fac77fffc75d6 2024-11-13T18:24:53,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a093e656f67e43d2952c03812a3ee002 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a093e656f67e43d2952c03812a3ee002 2024-11-13T18:24:53,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/84dfbbb26bcc4f198236411651e16a9d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/84dfbbb26bcc4f198236411651e16a9d 2024-11-13T18:24:53,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/16ddcb97959a488488fb674ee0855d41 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/16ddcb97959a488488fb674ee0855d41 2024-11-13T18:24:53,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a418432a87b542068883dccb483d0de1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/a418432a87b542068883dccb483d0de1 2024-11-13T18:24:53,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9fd94b6bcdae4fc0b89f1f2a883853d1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/9fd94b6bcdae4fc0b89f1f2a883853d1 2024-11-13T18:24:53,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/3abad75db26448a5b869cc2215b4529a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/3abad75db26448a5b869cc2215b4529a 2024-11-13T18:24:53,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/469bc2952f9c4465a0f8925c52a06704, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1eb46c700e8d46a5a82c6e782d63f181, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/6384f42ea378480b98e87d28fa6e3e3d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/ec99361e07f9471681e85d9bf4881e15, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f898ac289ea84bceb547496d90a28929, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9a79a8fb206e430fb4b078cc5e96b6af, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9e2ffe0c611f4a6896ed64fddd05a5b1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/15389914f17945a1bc73dcbb5977144b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/3b6a44e6b6084a64a99b3b5aa9d79331, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/493ccd447b9649ebba2caffdb18d38d7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c64e08d752014121b303d1a6e3a2f614, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/82c8b7cfa00a4d0c9c28bce36d3b443e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/e84ca77978b445b1b7bec18f77546f57, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/a832e99a68a74a43975d9f83ee8f9464, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c216beb03f3443c7bb5dc72d72ddec15, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/b5d761ee33e64577a2b99308c26b7d70, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/801ac13f664549928839c21393543e42, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/45f73acbb71144929af6fd1b3eb12d00, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/08b029c6bc1c4a19b859675d761e2ff1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/bfc597fd28c94a4fa957aae7e5877669, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eb3f4b1b04124d08afbbc3f6f759d51b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/421d49aeb67045a1b9e84b21dd6a6329, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eaf2f5873a5d43ac9321ae987a5bbfa3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/29cfebdbb5b44e65b354a82ddfd528e4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/7d5c54b3b3e04ef09f8a8da091ea4a56, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1d9187faca4341548b3b23f60b07a3d6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/453ae57b09ff4f168500c070f7a66944, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f27e803f6b224adcb1b3aa5ef008f536] to archive 2024-11-13T18:24:53,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:24:53,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/469bc2952f9c4465a0f8925c52a06704 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/469bc2952f9c4465a0f8925c52a06704 2024-11-13T18:24:53,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1eb46c700e8d46a5a82c6e782d63f181 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1eb46c700e8d46a5a82c6e782d63f181 2024-11-13T18:24:53,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/6384f42ea378480b98e87d28fa6e3e3d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/6384f42ea378480b98e87d28fa6e3e3d 2024-11-13T18:24:53,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/ec99361e07f9471681e85d9bf4881e15 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/ec99361e07f9471681e85d9bf4881e15 2024-11-13T18:24:53,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f898ac289ea84bceb547496d90a28929 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f898ac289ea84bceb547496d90a28929 2024-11-13T18:24:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T18:24:53,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9a79a8fb206e430fb4b078cc5e96b6af to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9a79a8fb206e430fb4b078cc5e96b6af 2024-11-13T18:24:53,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9e2ffe0c611f4a6896ed64fddd05a5b1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9e2ffe0c611f4a6896ed64fddd05a5b1 2024-11-13T18:24:53,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/15389914f17945a1bc73dcbb5977144b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/15389914f17945a1bc73dcbb5977144b 2024-11-13T18:24:53,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/3b6a44e6b6084a64a99b3b5aa9d79331 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/3b6a44e6b6084a64a99b3b5aa9d79331 2024-11-13T18:24:53,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/493ccd447b9649ebba2caffdb18d38d7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/493ccd447b9649ebba2caffdb18d38d7 2024-11-13T18:24:53,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c64e08d752014121b303d1a6e3a2f614 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c64e08d752014121b303d1a6e3a2f614 2024-11-13T18:24:53,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/82c8b7cfa00a4d0c9c28bce36d3b443e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/82c8b7cfa00a4d0c9c28bce36d3b443e 2024-11-13T18:24:53,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/e84ca77978b445b1b7bec18f77546f57 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/e84ca77978b445b1b7bec18f77546f57 2024-11-13T18:24:53,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/a832e99a68a74a43975d9f83ee8f9464 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/a832e99a68a74a43975d9f83ee8f9464 2024-11-13T18:24:53,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c216beb03f3443c7bb5dc72d72ddec15 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/c216beb03f3443c7bb5dc72d72ddec15 2024-11-13T18:24:53,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/b5d761ee33e64577a2b99308c26b7d70 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/b5d761ee33e64577a2b99308c26b7d70 2024-11-13T18:24:53,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/801ac13f664549928839c21393543e42 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/801ac13f664549928839c21393543e42 2024-11-13T18:24:53,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/45f73acbb71144929af6fd1b3eb12d00 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/45f73acbb71144929af6fd1b3eb12d00 2024-11-13T18:24:53,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/08b029c6bc1c4a19b859675d761e2ff1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/08b029c6bc1c4a19b859675d761e2ff1 2024-11-13T18:24:53,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/bfc597fd28c94a4fa957aae7e5877669 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/bfc597fd28c94a4fa957aae7e5877669 2024-11-13T18:24:53,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eb3f4b1b04124d08afbbc3f6f759d51b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eb3f4b1b04124d08afbbc3f6f759d51b 2024-11-13T18:24:53,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/421d49aeb67045a1b9e84b21dd6a6329 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/421d49aeb67045a1b9e84b21dd6a6329 2024-11-13T18:24:53,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eaf2f5873a5d43ac9321ae987a5bbfa3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/eaf2f5873a5d43ac9321ae987a5bbfa3 2024-11-13T18:24:53,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/29cfebdbb5b44e65b354a82ddfd528e4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/29cfebdbb5b44e65b354a82ddfd528e4 2024-11-13T18:24:53,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/7d5c54b3b3e04ef09f8a8da091ea4a56 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/7d5c54b3b3e04ef09f8a8da091ea4a56 2024-11-13T18:24:53,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1d9187faca4341548b3b23f60b07a3d6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/1d9187faca4341548b3b23f60b07a3d6 2024-11-13T18:24:53,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/453ae57b09ff4f168500c070f7a66944 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/453ae57b09ff4f168500c070f7a66944 2024-11-13T18:24:53,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f27e803f6b224adcb1b3aa5ef008f536 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/f27e803f6b224adcb1b3aa5ef008f536 2024-11-13T18:24:53,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62a263eef6f941c0b2a477de1991ea9b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e2a6f263ed3b4403b50b0d57fdd46467, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/955e98f1345b4cb688ac9677c9a76f82, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0bc4df3da9f347e786d9bb7dcdf6793b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4b18806df3f84f11854a7eddfb0ed1ab, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/688344bb28144b93882fac81692584ec, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d19f2fb239e5400abc7564934672f60a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/a6cc8dca719548a781036dc152b6972e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/6adcf2fe3a1c4c8c926f3556260e4fb6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cc375279685848e0aabb0ce5494e3ec3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ba484afefb7643de8d8f1f1338f23f2e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/7e5f28a154aa4fa2bc82c81bb36d3c6f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/3255881018034544bef39f33fff641f6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ff862b59f62e482cbc700d7cbfa455a0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/1e7af5cb78e64da5963334fd459ea872, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/67010b0e81ff47ba8d94fe237c11f2e1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d1d2379cc191406385a0b765165566d1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/df5e2fbe4ab24af9b2a44eef3d258d75, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4af20a11b12f473387518445d5981fd5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cde1eacd1105472e9f51279aea2b12b0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/2eb710f87c754efd949b30769f802a75, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/c84b64aefef74a84b9e5f614ff5a598f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e378ffccdfd742539b6fd6dba5929611, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/af5ef1475d86461bbfaebd3e26ffb0f5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62725bbc8efd4e2eada8dade3c49889b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/5d17d881f2244ffa936f28d3275425c2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4bbb569821904077a708bd22872b8ac3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0238344a7c7e4375a8fe1ac564f3c133] to archive 2024-11-13T18:24:53,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:24:53,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62a263eef6f941c0b2a477de1991ea9b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62a263eef6f941c0b2a477de1991ea9b 2024-11-13T18:24:53,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e2a6f263ed3b4403b50b0d57fdd46467 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e2a6f263ed3b4403b50b0d57fdd46467 2024-11-13T18:24:53,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/955e98f1345b4cb688ac9677c9a76f82 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/955e98f1345b4cb688ac9677c9a76f82 2024-11-13T18:24:53,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0bc4df3da9f347e786d9bb7dcdf6793b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0bc4df3da9f347e786d9bb7dcdf6793b 2024-11-13T18:24:53,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4b18806df3f84f11854a7eddfb0ed1ab to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4b18806df3f84f11854a7eddfb0ed1ab 2024-11-13T18:24:53,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/688344bb28144b93882fac81692584ec to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/688344bb28144b93882fac81692584ec 2024-11-13T18:24:53,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d19f2fb239e5400abc7564934672f60a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d19f2fb239e5400abc7564934672f60a 2024-11-13T18:24:53,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/a6cc8dca719548a781036dc152b6972e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/a6cc8dca719548a781036dc152b6972e 2024-11-13T18:24:53,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/6adcf2fe3a1c4c8c926f3556260e4fb6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/6adcf2fe3a1c4c8c926f3556260e4fb6 2024-11-13T18:24:53,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cc375279685848e0aabb0ce5494e3ec3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cc375279685848e0aabb0ce5494e3ec3 2024-11-13T18:24:53,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ba484afefb7643de8d8f1f1338f23f2e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ba484afefb7643de8d8f1f1338f23f2e 2024-11-13T18:24:53,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/7e5f28a154aa4fa2bc82c81bb36d3c6f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/7e5f28a154aa4fa2bc82c81bb36d3c6f 2024-11-13T18:24:53,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/3255881018034544bef39f33fff641f6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/3255881018034544bef39f33fff641f6 2024-11-13T18:24:53,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ff862b59f62e482cbc700d7cbfa455a0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/ff862b59f62e482cbc700d7cbfa455a0 2024-11-13T18:24:53,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/1e7af5cb78e64da5963334fd459ea872 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/1e7af5cb78e64da5963334fd459ea872 2024-11-13T18:24:53,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/67010b0e81ff47ba8d94fe237c11f2e1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/67010b0e81ff47ba8d94fe237c11f2e1 2024-11-13T18:24:54,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d1d2379cc191406385a0b765165566d1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/d1d2379cc191406385a0b765165566d1 2024-11-13T18:24:54,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/df5e2fbe4ab24af9b2a44eef3d258d75 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/df5e2fbe4ab24af9b2a44eef3d258d75 2024-11-13T18:24:54,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4af20a11b12f473387518445d5981fd5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4af20a11b12f473387518445d5981fd5 2024-11-13T18:24:54,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cde1eacd1105472e9f51279aea2b12b0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/cde1eacd1105472e9f51279aea2b12b0 2024-11-13T18:24:54,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/2eb710f87c754efd949b30769f802a75 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/2eb710f87c754efd949b30769f802a75 2024-11-13T18:24:54,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/c84b64aefef74a84b9e5f614ff5a598f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/c84b64aefef74a84b9e5f614ff5a598f 2024-11-13T18:24:54,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e378ffccdfd742539b6fd6dba5929611 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/e378ffccdfd742539b6fd6dba5929611 2024-11-13T18:24:54,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/af5ef1475d86461bbfaebd3e26ffb0f5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/af5ef1475d86461bbfaebd3e26ffb0f5 2024-11-13T18:24:54,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62725bbc8efd4e2eada8dade3c49889b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/62725bbc8efd4e2eada8dade3c49889b 2024-11-13T18:24:54,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/5d17d881f2244ffa936f28d3275425c2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/5d17d881f2244ffa936f28d3275425c2 2024-11-13T18:24:54,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4bbb569821904077a708bd22872b8ac3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/4bbb569821904077a708bd22872b8ac3 2024-11-13T18:24:54,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0238344a7c7e4375a8fe1ac564f3c133 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/0238344a7c7e4375a8fe1ac564f3c133 2024-11-13T18:24:54,236 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/recovered.edits/385.seqid, newMaxSeqId=385, maxSeqId=4 2024-11-13T18:24:54,243 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. 2024-11-13T18:24:54,243 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for a3e5c2fc3660c99b882b8a9ec24a558f: Waiting for close lock at 1731522291540Running coprocessor pre-close hooks at 1731522291540Disabling compacts and flushes for region at 1731522291540Disabling writes for close at 1731522291540Obtaining lock to block concurrent updates at 1731522291541 (+1 ms)Preparing flush snapshotting stores in a3e5c2fc3660c99b882b8a9ec24a558f at 1731522291541Finished memstore snapshotting TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731522291541Flushing stores of TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f. at 1731522291542 (+1 ms)Flushing a3e5c2fc3660c99b882b8a9ec24a558f/A: creating writer at 1731522291543 (+1 ms)Flushing mob file a3e5c2fc3660c99b882b8a9ec24a558f/A: appending metadata at 1731522291573 (+30 ms)Flushing mob file a3e5c2fc3660c99b882b8a9ec24a558f/A: closing flushed file at 1731522291573Flushing a3e5c2fc3660c99b882b8a9ec24a558f/A: appending metadata at 1731522292069 (+496 ms)Flushing a3e5c2fc3660c99b882b8a9ec24a558f/A: closing flushed file at 1731522292069Flushing a3e5c2fc3660c99b882b8a9ec24a558f/B: creating writer at 1731522292539 (+470 ms)Flushing a3e5c2fc3660c99b882b8a9ec24a558f/B: appending metadata at 1731522292548 (+9 ms)Flushing a3e5c2fc3660c99b882b8a9ec24a558f/B: closing flushed file at 1731522292548Flushing a3e5c2fc3660c99b882b8a9ec24a558f/C: creating writer at 1731522292649 (+101 ms)Flushing a3e5c2fc3660c99b882b8a9ec24a558f/C: appending metadata at 1731522292672 (+23 ms)Flushing a3e5c2fc3660c99b882b8a9ec24a558f/C: closing flushed file at 1731522292672Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e3f1eb9: reopening flushed file at 1731522293192 (+520 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e1a3c6e: reopening flushed file at 1731522293231 (+39 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6320c40e: reopening flushed file at 1731522293249 (+18 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a3e5c2fc3660c99b882b8a9ec24a558f in 1772ms, sequenceid=382, compaction requested=false at 1731522293312 (+63 ms)Writing region close event to WAL at 1731522294210 (+898 ms)Running coprocessor post-close hooks at 1731522294243 (+33 ms)Closed at 1731522294243 2024-11-13T18:24:54,247 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=a3e5c2fc3660c99b882b8a9ec24a558f, regionState=CLOSED 2024-11-13T18:24:54,250 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:54,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:24:54,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-13T18:24:54,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure a3e5c2fc3660c99b882b8a9ec24a558f, server=62ae331128c9,39615,1731522225114 in 2.8740 sec 2024-11-13T18:24:54,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-13T18:24:54,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3e5c2fc3660c99b882b8a9ec24a558f, UNASSIGN in 2.8890 sec 2024-11-13T18:24:54,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-13T18:24:54,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8960 sec 2024-11-13T18:24:54,276 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522294276"}]},"ts":"1731522294276"} 2024-11-13T18:24:54,284 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:24:54,284 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:24:54,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.9450 sec 2024-11-13T18:24:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T18:24:55,551 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:55,559 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:24:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:55,563 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:55,564 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T18:24:55,596 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,608 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/recovered.edits] 2024-11-13T18:24:55,612 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/14ea5d7bd67a488585a46d3a1c91e2bd to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/14ea5d7bd67a488585a46d3a1c91e2bd 2024-11-13T18:24:55,615 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bf3d6246fce444668fa4385dc209998d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/A/bf3d6246fce444668fa4385dc209998d 2024-11-13T18:24:55,619 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9b6cd0dabb034346b9bea297244a49b9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/9b6cd0dabb034346b9bea297244a49b9 2024-11-13T18:24:55,620 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/fd7b5ac105b84297b678c05dcc3c4f5e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/B/fd7b5ac105b84297b678c05dcc3c4f5e 2024-11-13T18:24:55,624 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/34a75a0aefb2443fa8e88aef2171a90b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/34a75a0aefb2443fa8e88aef2171a90b 2024-11-13T18:24:55,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/826c547d64a74edf94a00efdefe29630 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/C/826c547d64a74edf94a00efdefe29630 2024-11-13T18:24:55,635 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/recovered.edits/385.seqid to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f/recovered.edits/385.seqid 2024-11-13T18:24:55,636 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,638 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:24:55,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:24:55,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T18:24:55,677 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130a3ab5f9d3d545e1a81b165593918607_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130a3ab5f9d3d545e1a81b165593918607_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,680 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130e05540494194dd08fc452c9af86c836_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130e05540494194dd08fc452c9af86c836_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,683 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113253730d981a24104af1468c301ed679b_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113253730d981a24104af1468c301ed679b_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,685 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132ebb84fd95704e3683e976cc50f8d734_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132ebb84fd95704e3683e976cc50f8d734_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T18:24:55,693 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134c0be8c838434a9daea306c98c2a720e_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134c0be8c838434a9daea306c98c2a720e_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,699 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111359630d9c17ef4a87a1f23b62d93e713c_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111359630d9c17ef4a87a1f23b62d93e713c_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,701 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111362f42db0a055423a8cfdbb76ccfb7ae8_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111362f42db0a055423a8cfdbb76ccfb7ae8_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,704 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111364f1e6fcf3cf431b91acec4a70bad32f_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111364f1e6fcf3cf431b91acec4a70bad32f_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,706 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137935be3e047c4cc384922405e5dd3722_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137935be3e047c4cc384922405e5dd3722_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,711 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111379d8ea1bda214b608a19a6aa26364290_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111379d8ea1bda214b608a19a6aa26364290_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,718 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137acc58ae0b7c4b80a3a8f2f6e413ece4_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137acc58ae0b7c4b80a3a8f2f6e413ece4_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,725 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111387c2c3c5a3a4487ea242ed9461b50eba_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111387c2c3c5a3a4487ea242ed9461b50eba_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,730 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111388e06993cfcd4b1ea032086e3c8b8c90_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111388e06993cfcd4b1ea032086e3c8b8c90_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,744 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139a70d705e46a4ec295abb11d2657f771_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139a70d705e46a4ec295abb11d2657f771_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,762 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139dec8de5f8ff45a0a860ed8b7af4a0f0_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139dec8de5f8ff45a0a860ed8b7af4a0f0_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,765 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139e82d339b6b144618f797abd7f43c914_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139e82d339b6b144618f797abd7f43c914_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,767 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4c7fc69b6cb416794fe4863cc75d7b5_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4c7fc69b6cb416794fe4863cc75d7b5_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,769 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1014bb4b63d497cbd22629e3f2dec84_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1014bb4b63d497cbd22629e3f2dec84_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,771 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d911d8ef59f04d22a61874855b39ae1b_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d911d8ef59f04d22a61874855b39ae1b_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,774 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dce5a2074f6a4149a7f105381ff368a8_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dce5a2074f6a4149a7f105381ff368a8_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,791 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113faec04e2cd104740bd9d19885322af07_a3e5c2fc3660c99b882b8a9ec24a558f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113faec04e2cd104740bd9d19885322af07_a3e5c2fc3660c99b882b8a9ec24a558f 2024-11-13T18:24:55,793 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:24:55,797 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:55,801 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:24:55,804 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:24:55,806 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:55,806 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:24:55,806 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731522295806"}]},"ts":"9223372036854775807"} 2024-11-13T18:24:55,809 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:24:55,809 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a3e5c2fc3660c99b882b8a9ec24a558f, NAME => 'TestAcidGuarantees,,1731522263373.a3e5c2fc3660c99b882b8a9ec24a558f.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:24:55,809 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:24:55,809 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731522295809"}]},"ts":"9223372036854775807"} 2024-11-13T18:24:55,813 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:24:55,815 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:55,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 256 msec 2024-11-13T18:24:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T18:24:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:24:55,903 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:55,922 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247 (was 248), OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=2215 (was 2178) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1324 (was 855) - AvailableMemoryMB LEAK? - 2024-11-13T18:24:55,937 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=2215, ProcessCount=11, AvailableMemoryMB=1310 2024-11-13T18:24:55,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:24:55,941 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:24:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:24:55,946 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:24:55,946 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:55,947 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:24:55,949 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-13T18:24:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-13T18:24:55,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742056_1232 (size=1061) 2024-11-13T18:24:55,997 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b 2024-11-13T18:24:56,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742057_1233 (size=53) 2024-11-13T18:24:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-13T18:24:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-13T18:24:56,448 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:24:56,448 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 304012adc4150a48d7183d0af2e39078, disabling compactions & flushes 2024-11-13T18:24:56,448 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:56,448 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:56,448 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. after waiting 0 ms 2024-11-13T18:24:56,448 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:56,448 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:56,448 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 304012adc4150a48d7183d0af2e39078: Waiting for close lock at 1731522296448Disabling compacts and flushes for region at 1731522296448Disabling writes for close at 1731522296448Writing region close event to WAL at 1731522296448Closed at 1731522296448 2024-11-13T18:24:56,450 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:24:56,450 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731522296450"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731522296450"}]},"ts":"1731522296450"} 2024-11-13T18:24:56,458 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:24:56,460 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:24:56,466 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522296460"}]},"ts":"1731522296460"} 2024-11-13T18:24:56,475 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:24:56,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=304012adc4150a48d7183d0af2e39078, ASSIGN}] 2024-11-13T18:24:56,479 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=304012adc4150a48d7183d0af2e39078, ASSIGN 2024-11-13T18:24:56,480 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=304012adc4150a48d7183d0af2e39078, ASSIGN; state=OFFLINE, location=62ae331128c9,39615,1731522225114; forceNewPlan=false, retain=false 2024-11-13T18:24:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-13T18:24:56,631 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=304012adc4150a48d7183d0af2e39078, regionState=OPENING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:24:56,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=304012adc4150a48d7183d0af2e39078, ASSIGN because future has completed 2024-11-13T18:24:56,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:24:56,805 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:56,806 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:24:56,806 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,806 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:24:56,806 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,806 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,809 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,810 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:24:56,811 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 304012adc4150a48d7183d0af2e39078 columnFamilyName A 2024-11-13T18:24:56,811 DEBUG [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:56,813 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] regionserver.HStore(327): Store=304012adc4150a48d7183d0af2e39078/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:24:56,813 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,816 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:24:56,817 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 304012adc4150a48d7183d0af2e39078 columnFamilyName B 2024-11-13T18:24:56,817 DEBUG [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:56,823 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] regionserver.HStore(327): Store=304012adc4150a48d7183d0af2e39078/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:24:56,823 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,829 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:24:56,830 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 304012adc4150a48d7183d0af2e39078 columnFamilyName C 2024-11-13T18:24:56,830 DEBUG [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:24:56,832 INFO [StoreOpener-304012adc4150a48d7183d0af2e39078-1 {}] regionserver.HStore(327): Store=304012adc4150a48d7183d0af2e39078/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:24:56,832 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:56,832 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,836 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,836 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,841 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,841 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,843 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:24:56,845 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,864 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:24:56,865 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 304012adc4150a48d7183d0af2e39078; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65008659, jitterRate=-0.03129549324512482}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:24:56,865 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:56,866 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 304012adc4150a48d7183d0af2e39078: Running coprocessor pre-open hook at 1731522296806Writing region info on filesystem at 1731522296806Initializing all the Stores at 1731522296807 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522296808 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522296808Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522296808Cleaning up temporary data from old regions at 1731522296841 (+33 ms)Running coprocessor post-open hooks at 1731522296865 (+24 ms)Region opened successfully at 1731522296866 (+1 ms) 2024-11-13T18:24:56,875 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., pid=57, masterSystemTime=1731522296790 2024-11-13T18:24:56,878 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:56,878 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:56,880 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=304012adc4150a48d7183d0af2e39078, regionState=OPEN, openSeqNum=2, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:24:56,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:24:56,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-13T18:24:56,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 in 262 msec 2024-11-13T18:24:56,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-13T18:24:56,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=304012adc4150a48d7183d0af2e39078, ASSIGN in 422 msec 2024-11-13T18:24:56,901 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:24:56,901 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522296901"}]},"ts":"1731522296901"} 2024-11-13T18:24:56,903 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:24:56,905 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:24:56,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 964 msec 2024-11-13T18:24:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-13T18:24:57,101 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:24:57,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@734f06ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,105 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@529bd45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,114 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549dd024, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,116 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,123 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:24:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 62ae331128c9,45483,1731522221514 2024-11-13T18:24:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@546a558d 2024-11-13T18:24:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:24:57,126 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:24:57,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23e54de5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,131 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177f18c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,134 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd5ba5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f761ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,149 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3093713e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,152 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a41ee21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48787f5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,161 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dc28f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,164 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d32fcf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2715d903, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,179 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,185 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a683549, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,190 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a525153, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478d996b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,216 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6636f739, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,218 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b3272a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333e8b94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,222 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f69b233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,225 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22cb5e37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d142ea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,244 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d762d90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,248 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1466bf2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f02932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,269 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a896bd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,275 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@409ed80c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e297f18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,281 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d587ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,284 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@967f717, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3455134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:24:57,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:24:57,308 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:24:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:24:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:24:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9c34f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:24:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:24:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:24:57,315 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:24:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d124c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:24:57,320 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,321 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:24:57,331 DEBUG [Thread-1110 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,331 DEBUG [Thread-1110 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:24:57,350 DEBUG [Thread-1105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,351 DEBUG [Thread-1105 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,351 DEBUG [Thread-1105 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,361 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,363 DEBUG [Thread-1108 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,367 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:24:57,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:24:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,382 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:57,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:24:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:24:57,390 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:24:57,392 DEBUG [Thread-1116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:24:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:24:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:57,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:24:57,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:57,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:24:57,398 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,399 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:24:57,403 DEBUG [Thread-1099 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-13T18:24:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:24:57,420 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,423 DEBUG [Thread-1099 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,423 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:24:57,425 DEBUG [Thread-1114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:24:57,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:24:57,426 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:24:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:24:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:24:57,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:24:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:24:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:24:57,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:24:57,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/d2753ef05c884baa8cc7ab04ce36b74d is 50, key is test_row_0/A:col10/1731522297386/Put/seqid=0 2024-11-13T18:24:57,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522357460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522357463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522357484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522357502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522357504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:24:57,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:24:57,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:57,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:24:57,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:57,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:57,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:57,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742058_1234 (size=12001) 2024-11-13T18:24:57,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/d2753ef05c884baa8cc7ab04ce36b74d 2024-11-13T18:24:57,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522357606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522357607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522357621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522357626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522357631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/60cfd8578fa4458095d28d184f022b44 is 50, key is test_row_0/B:col10/1731522297386/Put/seqid=0 2024-11-13T18:24:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:24:57,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:24:57,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:57,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:24:57,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:57,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:57,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:57,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742059_1235 (size=12001) 2024-11-13T18:24:57,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522357825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522357825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522357836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522357846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522357846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:57,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:24:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:24:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:57,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:57,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:24:58,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:24:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:24:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522358147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:58,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522358147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:58,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:58,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/60cfd8578fa4458095d28d184f022b44 2024-11-13T18:24:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522358148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:58,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522358167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:58,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522358179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:58,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:24:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:24:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/934e20589a0b4374b4b30834f0f13c36 is 50, key is test_row_0/C:col10/1731522297386/Put/seqid=0 2024-11-13T18:24:58,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742060_1236 (size=12001) 2024-11-13T18:24:58,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:24:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:24:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:24:58,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:24:58,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:24:58,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522358683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:58,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522358683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:58,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522358686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522358690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:58,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:24:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522358691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:24:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:58,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:24:58,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:24:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:24:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:24:58,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:24:58,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/934e20589a0b4374b4b30834f0f13c36 2024-11-13T18:24:58,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/d2753ef05c884baa8cc7ab04ce36b74d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d2753ef05c884baa8cc7ab04ce36b74d 2024-11-13T18:24:58,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d2753ef05c884baa8cc7ab04ce36b74d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:24:58,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/60cfd8578fa4458095d28d184f022b44 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/60cfd8578fa4458095d28d184f022b44 2024-11-13T18:24:58,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/60cfd8578fa4458095d28d184f022b44, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:24:58,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/934e20589a0b4374b4b30834f0f13c36 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/934e20589a0b4374b4b30834f0f13c36 2024-11-13T18:24:58,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/934e20589a0b4374b4b30834f0f13c36, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:24:58,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 304012adc4150a48d7183d0af2e39078 in 1418ms, sequenceid=13, compaction requested=false 2024-11-13T18:24:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:24:58,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:24:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:58,860 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:24:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:24:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:24:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:24:58,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:58,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/9a701f9a1c914634a16b96d5170b48a2 is 50, key is test_row_0/A:col10/1731522297493/Put/seqid=0 2024-11-13T18:24:58,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742061_1237 (size=12001) 2024-11-13T18:24:58,929 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/9a701f9a1c914634a16b96d5170b48a2 2024-11-13T18:24:58,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/70828f7be70a4a92ba5b580b0f0bb12f is 50, key is test_row_0/B:col10/1731522297493/Put/seqid=0 2024-11-13T18:24:59,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742062_1238 (size=12001) 2024-11-13T18:24:59,459 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/70828f7be70a4a92ba5b580b0f0bb12f 2024-11-13T18:24:59,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/8fb8d5e84694434dad284ef3f5311c6a is 50, key is test_row_0/C:col10/1731522297493/Put/seqid=0 2024-11-13T18:24:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:24:59,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742063_1239 (size=12001) 2024-11-13T18:24:59,594 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/8fb8d5e84694434dad284ef3f5311c6a 2024-11-13T18:24:59,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/9a701f9a1c914634a16b96d5170b48a2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a701f9a1c914634a16b96d5170b48a2 2024-11-13T18:24:59,607 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a701f9a1c914634a16b96d5170b48a2, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T18:24:59,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/70828f7be70a4a92ba5b580b0f0bb12f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/70828f7be70a4a92ba5b580b0f0bb12f 2024-11-13T18:24:59,629 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/70828f7be70a4a92ba5b580b0f0bb12f, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T18:24:59,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/8fb8d5e84694434dad284ef3f5311c6a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8fb8d5e84694434dad284ef3f5311c6a 2024-11-13T18:24:59,638 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8fb8d5e84694434dad284ef3f5311c6a, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T18:24:59,643 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 304012adc4150a48d7183d0af2e39078 in 783ms, sequenceid=33, compaction requested=false 2024-11-13T18:24:59,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:24:59,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:24:59,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T18:24:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T18:24:59,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T18:24:59,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2210 sec 2024-11-13T18:24:59,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.2660 sec 2024-11-13T18:24:59,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:24:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:24:59,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:24:59,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:59,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:24:59,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:59,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:24:59,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:24:59,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/ea9eeca33c7945c5a1b4e649e5567ee3 is 50, key is test_row_0/A:col10/1731522299717/Put/seqid=0 2024-11-13T18:24:59,825 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:24:59,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742064_1240 (size=12001) 2024-11-13T18:24:59,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/ea9eeca33c7945c5a1b4e649e5567ee3 2024-11-13T18:24:59,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/fb4b69f4945643f98f046cebe943d176 is 50, key is test_row_0/B:col10/1731522299717/Put/seqid=0 2024-11-13T18:24:59,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522299813, max timestamp=1731522299890Num uniques 859; 2024-11-13T18:24:59,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:59,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522299813, max timestamp=1731522299890Num uniques 859; 2024-11-13T18:24:59,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:24:59,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522299813, max timestamp=1731522299890Num uniques 859; 2024-11-13T18:24:59,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:24:59,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742065_1241 (size=12001) 2024-11-13T18:24:59,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/fb4b69f4945643f98f046cebe943d176 2024-11-13T18:24:59,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/1d1315091ccc40df808c3843d543097c is 50, key is test_row_0/C:col10/1731522299717/Put/seqid=0 2024-11-13T18:25:00,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522359971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522359991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522359993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522359997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522360001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742066_1242 (size=12001) 2024-11-13T18:25:00,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/1d1315091ccc40df808c3843d543097c 2024-11-13T18:25:00,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/ea9eeca33c7945c5a1b4e649e5567ee3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/ea9eeca33c7945c5a1b4e649e5567ee3 2024-11-13T18:25:00,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/ea9eeca33c7945c5a1b4e649e5567ee3, entries=150, sequenceid=43, filesize=11.7 K 2024-11-13T18:25:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/fb4b69f4945643f98f046cebe943d176 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/fb4b69f4945643f98f046cebe943d176 2024-11-13T18:25:00,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/fb4b69f4945643f98f046cebe943d176, entries=150, sequenceid=43, filesize=11.7 K 2024-11-13T18:25:00,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/1d1315091ccc40df808c3843d543097c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d1315091ccc40df808c3843d543097c 2024-11-13T18:25:00,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d1315091ccc40df808c3843d543097c, entries=150, sequenceid=43, filesize=11.7 K 2024-11-13T18:25:00,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 304012adc4150a48d7183d0af2e39078 in 350ms, sequenceid=43, compaction requested=true 2024-11-13T18:25:00,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:00,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:00,099 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:00,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:00,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:00,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:00,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:00,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:00,100 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:00,100 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/A is initiating minor compaction (all files) 2024-11-13T18:25:00,100 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/A in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:00,101 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:00,101 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d2753ef05c884baa8cc7ab04ce36b74d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a701f9a1c914634a16b96d5170b48a2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/ea9eeca33c7945c5a1b4e649e5567ee3] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=35.2 K 2024-11-13T18:25:00,102 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting d2753ef05c884baa8cc7ab04ce36b74d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731522297377 2024-11-13T18:25:00,103 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a701f9a1c914634a16b96d5170b48a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731522297445 2024-11-13T18:25:00,103 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting ea9eeca33c7945c5a1b4e649e5567ee3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731522299717 2024-11-13T18:25:00,115 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:00,115 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/B is initiating minor compaction (all files) 2024-11-13T18:25:00,115 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/B in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:00,115 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/60cfd8578fa4458095d28d184f022b44, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/70828f7be70a4a92ba5b580b0f0bb12f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/fb4b69f4945643f98f046cebe943d176] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=35.2 K 2024-11-13T18:25:00,116 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60cfd8578fa4458095d28d184f022b44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731522297377 2024-11-13T18:25:00,123 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70828f7be70a4a92ba5b580b0f0bb12f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731522297445 2024-11-13T18:25:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:00,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=140.49 KB heapSize=404.70 KB 2024-11-13T18:25:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:25:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:25:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:25:00,128 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb4b69f4945643f98f046cebe943d176, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731522299717 2024-11-13T18:25:00,128 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#A#compaction#195 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:00,128 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/947f24a9061c4462bda7a11d1c4a046f is 50, key is test_row_0/A:col10/1731522299717/Put/seqid=0 2024-11-13T18:25:00,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/a74f682396764abba595a96b87de0b33 is 50, key is test_row_0/A:col10/1731522300120/Put/seqid=0 2024-11-13T18:25:00,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522360162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522360162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522360162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522360165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742067_1243 (size=12104) 2024-11-13T18:25:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522360195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,213 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#B#compaction#197 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:00,214 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/d9a48ef16d254de09c527082c0326a20 is 50, key is test_row_0/B:col10/1731522299717/Put/seqid=0 2024-11-13T18:25:00,224 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/947f24a9061c4462bda7a11d1c4a046f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/947f24a9061c4462bda7a11d1c4a046f 2024-11-13T18:25:00,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742068_1244 (size=14341) 2024-11-13T18:25:00,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/a74f682396764abba595a96b87de0b33 2024-11-13T18:25:00,241 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/A of 304012adc4150a48d7183d0af2e39078 into 947f24a9061c4462bda7a11d1c4a046f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:00,241 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:00,241 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/A, priority=13, startTime=1731522300098; duration=0sec 2024-11-13T18:25:00,241 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:00,241 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:00,241 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:00,244 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:00,244 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/C is initiating minor compaction (all files) 2024-11-13T18:25:00,244 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/C in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:00,244 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/934e20589a0b4374b4b30834f0f13c36, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8fb8d5e84694434dad284ef3f5311c6a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d1315091ccc40df808c3843d543097c] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=35.2 K 2024-11-13T18:25:00,245 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 934e20589a0b4374b4b30834f0f13c36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731522297377 2024-11-13T18:25:00,246 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb8d5e84694434dad284ef3f5311c6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731522297445 2024-11-13T18:25:00,246 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d1315091ccc40df808c3843d543097c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731522299717 2024-11-13T18:25:00,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742069_1245 (size=12104) 2024-11-13T18:25:00,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/ef074afe7de44fe68e9743ac99a88cc9 is 50, key is test_row_0/B:col10/1731522300120/Put/seqid=0 2024-11-13T18:25:00,300 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/d9a48ef16d254de09c527082c0326a20 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d9a48ef16d254de09c527082c0326a20 2024-11-13T18:25:00,310 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#C#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:00,311 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/dc2217d663284a529334c4bd0bb5a05c is 50, key is test_row_0/C:col10/1731522299717/Put/seqid=0 2024-11-13T18:25:00,328 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/B of 304012adc4150a48d7183d0af2e39078 into d9a48ef16d254de09c527082c0326a20(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:00,328 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:00,328 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/B, priority=13, startTime=1731522300099; duration=0sec 2024-11-13T18:25:00,329 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:00,329 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:00,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522360309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522360310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522360310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522360316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522360321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742070_1246 (size=12001) 2024-11-13T18:25:00,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/ef074afe7de44fe68e9743ac99a88cc9 2024-11-13T18:25:00,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/3de65910a5f5440e8edd83f053ec9cc8 is 50, key is test_row_0/C:col10/1731522300120/Put/seqid=0 2024-11-13T18:25:00,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742071_1247 (size=12104) 2024-11-13T18:25:00,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742072_1248 (size=12001) 2024-11-13T18:25:00,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522360547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522360548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522360549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522360550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522360547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,874 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/dc2217d663284a529334c4bd0bb5a05c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/dc2217d663284a529334c4bd0bb5a05c 2024-11-13T18:25:00,882 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/C of 304012adc4150a48d7183d0af2e39078 into dc2217d663284a529334c4bd0bb5a05c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:00,882 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:00,882 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/C, priority=13, startTime=1731522300099; duration=0sec 2024-11-13T18:25:00,882 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:00,882 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:00,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522360877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522360877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522360877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522360877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522360877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:00,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/3de65910a5f5440e8edd83f053ec9cc8 2024-11-13T18:25:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:00,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/a74f682396764abba595a96b87de0b33 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/a74f682396764abba595a96b87de0b33 2024-11-13T18:25:00,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/a74f682396764abba595a96b87de0b33, entries=200, sequenceid=81, filesize=14.0 K 2024-11-13T18:25:00,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/ef074afe7de44fe68e9743ac99a88cc9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ef074afe7de44fe68e9743ac99a88cc9 2024-11-13T18:25:00,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ef074afe7de44fe68e9743ac99a88cc9, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T18:25:00,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/3de65910a5f5440e8edd83f053ec9cc8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/3de65910a5f5440e8edd83f053ec9cc8 2024-11-13T18:25:00,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/3de65910a5f5440e8edd83f053ec9cc8, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T18:25:00,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.49 KB/143865, heapSize ~404.65 KB/414360, currentSize=40.25 KB/41220 for 304012adc4150a48d7183d0af2e39078 in 843ms, sequenceid=81, compaction requested=false 2024-11-13T18:25:00,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:01,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:01,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:01,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:01,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:01,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/187460c88b9141e7a6d01de97eadcaf9 is 50, key is test_row_1/A:col10/1731522300179/Put/seqid=0 2024-11-13T18:25:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742073_1249 (size=9657) 2024-11-13T18:25:01,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522361499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522301483, max timestamp=1731522301507Num uniques 859; 2024-11-13T18:25:01,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:25:01,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522301483, max timestamp=1731522301507Num uniques 859; 2024-11-13T18:25:01,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:25:01,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522361565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522361565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:25:01,582 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:01,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522361573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522361580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,589 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-13T18:25:01,593 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:25:01,599 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:01,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:01,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522361626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522361676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522361687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:25:01,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522361696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522361696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:25:01,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:01,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:01,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:01,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:01,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522361851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/187460c88b9141e7a6d01de97eadcaf9 2024-11-13T18:25:01,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/47f78cc7864e4641910cc60c4a958907 is 50, key is test_row_1/B:col10/1731522300179/Put/seqid=0 2024-11-13T18:25:01,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522361896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:25:01,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:25:01,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:01,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:01,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:01,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:01,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:01,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522361916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522361916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522361926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:01,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742074_1250 (size=9657) 2024-11-13T18:25:02,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:25:02,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:02,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522362167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:02,206 INFO [AsyncFSWAL-0-hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData-prefix:62ae331128c9,45483,1731522221514 {}] wal.AbstractFSWAL(1368): Slow sync cost: 102 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46535,DS-cab5a945-efa3-4827-b04b-1ecd7edf5cb9,DISK]] 2024-11-13T18:25:02,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:25:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522362225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:02,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522362239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:02,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:02,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522362263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522362264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:02,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:25:02,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:02,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/47f78cc7864e4641910cc60c4a958907 2024-11-13T18:25:02,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/23eb4e7bb7384f0e991934f8dc9d4b76 is 50, key is test_row_1/C:col10/1731522300179/Put/seqid=0 2024-11-13T18:25:02,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742075_1251 (size=9657) 2024-11-13T18:25:02,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:25:02,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:02,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:25:02,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522362691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:25:02,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522362758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:02,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522362766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:02,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522362782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:02,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:02,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522362783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:02,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:02,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:25:02,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:02,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:02,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/23eb4e7bb7384f0e991934f8dc9d4b76 2024-11-13T18:25:02,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/187460c88b9141e7a6d01de97eadcaf9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/187460c88b9141e7a6d01de97eadcaf9 2024-11-13T18:25:02,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/187460c88b9141e7a6d01de97eadcaf9, entries=100, sequenceid=94, filesize=9.4 K 2024-11-13T18:25:02,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/47f78cc7864e4641910cc60c4a958907 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/47f78cc7864e4641910cc60c4a958907 2024-11-13T18:25:02,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/47f78cc7864e4641910cc60c4a958907, entries=100, sequenceid=94, filesize=9.4 K 2024-11-13T18:25:02,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/23eb4e7bb7384f0e991934f8dc9d4b76 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/23eb4e7bb7384f0e991934f8dc9d4b76 2024-11-13T18:25:02,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/23eb4e7bb7384f0e991934f8dc9d4b76, entries=100, sequenceid=94, filesize=9.4 K 2024-11-13T18:25:02,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.15 KB/134300 for 304012adc4150a48d7183d0af2e39078 in 1570ms, sequenceid=94, compaction requested=true 2024-11-13T18:25:02,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:02,977 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:02,983 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:02,983 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/A is initiating minor compaction (all files) 2024-11-13T18:25:02,984 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/A in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,984 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/947f24a9061c4462bda7a11d1c4a046f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/a74f682396764abba595a96b87de0b33, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/187460c88b9141e7a6d01de97eadcaf9] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=35.3 K 2024-11-13T18:25:02,984 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 947f24a9061c4462bda7a11d1c4a046f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731522299717 2024-11-13T18:25:02,985 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting a74f682396764abba595a96b87de0b33, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731522299989 2024-11-13T18:25:02,985 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 187460c88b9141e7a6d01de97eadcaf9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731522300179 2024-11-13T18:25:02,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:02,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:02,996 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:02,997 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:02,997 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/B is initiating minor compaction (all files) 2024-11-13T18:25:02,997 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/B in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:02,997 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d9a48ef16d254de09c527082c0326a20, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ef074afe7de44fe68e9743ac99a88cc9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/47f78cc7864e4641910cc60c4a958907] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=33.0 K 2024-11-13T18:25:02,999 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a48ef16d254de09c527082c0326a20, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731522299717 2024-11-13T18:25:03,002 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:03,002 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/dd676c2fd41f4af1aa0cc481a02fdc24 is 50, key is test_row_0/A:col10/1731522300120/Put/seqid=0 2024-11-13T18:25:03,007 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting ef074afe7de44fe68e9743ac99a88cc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731522299989 2024-11-13T18:25:03,008 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f78cc7864e4641910cc60c4a958907, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731522300179 2024-11-13T18:25:03,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:25:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:03,020 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=131.15 KB heapSize=381.48 KB 2024-11-13T18:25:03,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:03,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:03,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:03,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:03,025 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#B#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:03,026 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a488f206979d474bbd6c51de1aaffcc3 is 50, key is test_row_0/B:col10/1731522300120/Put/seqid=0 2024-11-13T18:25:03,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742077_1253 (size=12207) 2024-11-13T18:25:03,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/8efaa9d9cb4d4fcd8588315441639d0b is 50, key is test_row_0/A:col10/1731522301526/Put/seqid=0 2024-11-13T18:25:03,091 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a488f206979d474bbd6c51de1aaffcc3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a488f206979d474bbd6c51de1aaffcc3 2024-11-13T18:25:03,103 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/B of 304012adc4150a48d7183d0af2e39078 into a488f206979d474bbd6c51de1aaffcc3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:03,103 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:03,104 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/B, priority=13, startTime=1731522302996; duration=0sec 2024-11-13T18:25:03,104 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:03,104 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:03,104 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:03,112 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:03,112 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/C is initiating minor compaction (all files) 2024-11-13T18:25:03,113 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/C in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:03,113 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/dc2217d663284a529334c4bd0bb5a05c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/3de65910a5f5440e8edd83f053ec9cc8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/23eb4e7bb7384f0e991934f8dc9d4b76] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=33.0 K 2024-11-13T18:25:03,113 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting dc2217d663284a529334c4bd0bb5a05c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731522299717 2024-11-13T18:25:03,115 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de65910a5f5440e8edd83f053ec9cc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731522299989 2024-11-13T18:25:03,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742076_1252 (size=12207) 2024-11-13T18:25:03,121 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 23eb4e7bb7384f0e991934f8dc9d4b76, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731522300179 2024-11-13T18:25:03,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742078_1254 (size=12001) 2024-11-13T18:25:03,179 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.15 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/8efaa9d9cb4d4fcd8588315441639d0b 2024-11-13T18:25:03,184 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/dd676c2fd41f4af1aa0cc481a02fdc24 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/dd676c2fd41f4af1aa0cc481a02fdc24 2024-11-13T18:25:03,217 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#C#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:03,218 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/1d3cd5f9796b4dfb9c315a370be98ecb is 50, key is test_row_0/C:col10/1731522300120/Put/seqid=0 2024-11-13T18:25:03,223 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/A of 304012adc4150a48d7183d0af2e39078 into dd676c2fd41f4af1aa0cc481a02fdc24(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:03,223 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:03,223 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/A, priority=13, startTime=1731522302977; duration=0sec 2024-11-13T18:25:03,223 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:03,223 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/414edb91886843bd8d0a75b36dcc4446 is 50, key is test_row_0/B:col10/1731522301526/Put/seqid=0 2024-11-13T18:25:03,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742079_1255 (size=12207) 2024-11-13T18:25:03,311 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/1d3cd5f9796b4dfb9c315a370be98ecb as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d3cd5f9796b4dfb9c315a370be98ecb 2024-11-13T18:25:03,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742080_1256 (size=12001) 2024-11-13T18:25:03,315 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.15 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/414edb91886843bd8d0a75b36dcc4446 2024-11-13T18:25:03,335 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/C of 304012adc4150a48d7183d0af2e39078 into 1d3cd5f9796b4dfb9c315a370be98ecb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:03,335 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:03,335 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/C, priority=13, startTime=1731522303020; duration=0sec 2024-11-13T18:25:03,335 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:03,335 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:03,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/279066480573440c9c24af55f3708d1e is 50, key is test_row_0/C:col10/1731522301526/Put/seqid=0 2024-11-13T18:25:03,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742081_1257 (size=12001) 2024-11-13T18:25:03,377 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/279066480573440c9c24af55f3708d1e 2024-11-13T18:25:03,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/8efaa9d9cb4d4fcd8588315441639d0b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/8efaa9d9cb4d4fcd8588315441639d0b 2024-11-13T18:25:03,392 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/8efaa9d9cb4d4fcd8588315441639d0b, entries=150, sequenceid=126, filesize=11.7 K 2024-11-13T18:25:03,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/414edb91886843bd8d0a75b36dcc4446 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/414edb91886843bd8d0a75b36dcc4446 2024-11-13T18:25:03,410 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/414edb91886843bd8d0a75b36dcc4446, entries=150, sequenceid=126, filesize=11.7 K 2024-11-13T18:25:03,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/279066480573440c9c24af55f3708d1e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/279066480573440c9c24af55f3708d1e 2024-11-13T18:25:03,440 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/279066480573440c9c24af55f3708d1e, entries=150, sequenceid=126, filesize=11.7 K 2024-11-13T18:25:03,441 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~131.15 KB/134300, heapSize ~381.44 KB/390592, currentSize=0 B/0 for 304012adc4150a48d7183d0af2e39078 in 421ms, sequenceid=126, compaction requested=false 2024-11-13T18:25:03,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:03,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:03,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-13T18:25:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-13T18:25:03,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T18:25:03,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8440 sec 2024-11-13T18:25:03,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.8570 sec 2024-11-13T18:25:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:25:03,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:03,772 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-13T18:25:03,775 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:25:03,777 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:03,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:03,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:03,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/e28f6c5d5d8649e5b67224c441e20ad9 is 50, key is test_row_0/A:col10/1731522303793/Put/seqid=0 2024-11-13T18:25:03,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742082_1258 (size=12151) 2024-11-13T18:25:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:25:03,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/e28f6c5d5d8649e5b67224c441e20ad9 2024-11-13T18:25:03,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/efaa42408e9f422886b61b11f8233b90 is 50, key is test_row_0/B:col10/1731522303793/Put/seqid=0 2024-11-13T18:25:03,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522303898, max timestamp=1731522303913Num uniques 859; 2024-11-13T18:25:03,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:03,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522303898, max timestamp=1731522303913Num uniques 859; 2024-11-13T18:25:03,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:03,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522303898, max timestamp=1731522303913Num uniques 859; 2024-11-13T18:25:03,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:03,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:03,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:03,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:03,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:03,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:03,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:03,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742083_1259 (size=12151) 2024-11-13T18:25:04,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522364006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522364010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522364003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522364016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522364030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:25:04,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522364136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522364139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522364146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522364156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522364166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:04,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:04,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522364361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522364363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522364368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522364390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522364390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:25:04,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/efaa42408e9f422886b61b11f8233b90 2024-11-13T18:25:04,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:04,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:04,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/4b8eb246bcc94a6c9e09b166ee325e4a is 50, key is test_row_0/C:col10/1731522303793/Put/seqid=0 2024-11-13T18:25:04,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742084_1260 (size=12151) 2024-11-13T18:25:04,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/4b8eb246bcc94a6c9e09b166ee325e4a 2024-11-13T18:25:04,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/e28f6c5d5d8649e5b67224c441e20ad9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/e28f6c5d5d8649e5b67224c441e20ad9 2024-11-13T18:25:04,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:04,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/e28f6c5d5d8649e5b67224c441e20ad9, entries=150, sequenceid=139, filesize=11.9 K 2024-11-13T18:25:04,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/efaa42408e9f422886b61b11f8233b90 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/efaa42408e9f422886b61b11f8233b90 2024-11-13T18:25:04,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/efaa42408e9f422886b61b11f8233b90, entries=150, sequenceid=139, filesize=11.9 K 2024-11-13T18:25:04,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/4b8eb246bcc94a6c9e09b166ee325e4a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b8eb246bcc94a6c9e09b166ee325e4a 2024-11-13T18:25:04,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b8eb246bcc94a6c9e09b166ee325e4a, entries=150, sequenceid=139, filesize=11.9 K 2024-11-13T18:25:04,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 304012adc4150a48d7183d0af2e39078 in 818ms, sequenceid=139, compaction requested=true 2024-11-13T18:25:04,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:04,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:04,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:04,630 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:04,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:04,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:04,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:04,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:04,630 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:04,634 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:04,635 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/A is initiating minor compaction (all files) 2024-11-13T18:25:04,635 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/A in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,635 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/dd676c2fd41f4af1aa0cc481a02fdc24, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/8efaa9d9cb4d4fcd8588315441639d0b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/e28f6c5d5d8649e5b67224c441e20ad9] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=35.5 K 2024-11-13T18:25:04,635 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:04,635 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/B is initiating minor compaction (all files) 2024-11-13T18:25:04,635 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/B in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,635 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a488f206979d474bbd6c51de1aaffcc3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/414edb91886843bd8d0a75b36dcc4446, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/efaa42408e9f422886b61b11f8233b90] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=35.5 K 2024-11-13T18:25:04,636 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd676c2fd41f4af1aa0cc481a02fdc24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731522300120 2024-11-13T18:25:04,636 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting a488f206979d474bbd6c51de1aaffcc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731522300120 2024-11-13T18:25:04,636 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 414edb91886843bd8d0a75b36dcc4446, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731522301526 2024-11-13T18:25:04,636 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8efaa9d9cb4d4fcd8588315441639d0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731522301526 2024-11-13T18:25:04,637 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting e28f6c5d5d8649e5b67224c441e20ad9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731522303757 2024-11-13T18:25:04,637 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting efaa42408e9f422886b61b11f8233b90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731522303757 2024-11-13T18:25:04,672 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:04,673 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5d67cef49e9a41c6bb0546760f172c98 is 50, key is test_row_0/A:col10/1731522303793/Put/seqid=0 2024-11-13T18:25:04,676 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:04,677 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/f39d83cdb1414f4780f57533296aa890 is 50, key is test_row_0/B:col10/1731522303793/Put/seqid=0 2024-11-13T18:25:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:04,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-13T18:25:04,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:04,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:04,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:04,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:04,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:04,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:04,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/cfb7479b134f41b8abd8c6b7aec76b3f is 50, key is test_row_0/A:col10/1731522304693/Put/seqid=0 2024-11-13T18:25:04,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742086_1262 (size=12459) 2024-11-13T18:25:04,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742085_1261 (size=12459) 2024-11-13T18:25:04,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522364722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522364724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522364729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:04,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522364742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522364745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742087_1263 (size=14541) 2024-11-13T18:25:04,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/cfb7479b134f41b8abd8c6b7aec76b3f 2024-11-13T18:25:04,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/4c0bb97af17c48538cf1332d375a0a87 is 50, key is test_row_0/B:col10/1731522304693/Put/seqid=0 2024-11-13T18:25:04,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522364859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522364866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522364866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522364859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742088_1264 (size=12151) 2024-11-13T18:25:04,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522364887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:04,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:25:04,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:04,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:04,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:04,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:05,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:05,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:05,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:05,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:05,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:05,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:05,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522365086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:05,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522365086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:05,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522365087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:05,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522365096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:05,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:05,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522365116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:05,151 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5d67cef49e9a41c6bb0546760f172c98 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5d67cef49e9a41c6bb0546760f172c98 2024-11-13T18:25:05,159 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/f39d83cdb1414f4780f57533296aa890 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/f39d83cdb1414f4780f57533296aa890 2024-11-13T18:25:05,190 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/A of 304012adc4150a48d7183d0af2e39078 into 5d67cef49e9a41c6bb0546760f172c98(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:05,190 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:05,190 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/A, priority=13, startTime=1731522304630; duration=0sec 2024-11-13T18:25:05,190 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:05,190 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:05,190 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:05,192 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/B of 304012adc4150a48d7183d0af2e39078 into f39d83cdb1414f4780f57533296aa890(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:05,192 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:05,192 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/B, priority=13, startTime=1731522304630; duration=0sec 2024-11-13T18:25:05,192 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:05,192 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:05,198 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:05,198 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/C is initiating minor compaction (all files) 2024-11-13T18:25:05,198 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/C in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:05,198 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d3cd5f9796b4dfb9c315a370be98ecb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/279066480573440c9c24af55f3708d1e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b8eb246bcc94a6c9e09b166ee325e4a] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=35.5 K 2024-11-13T18:25:05,199 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d3cd5f9796b4dfb9c315a370be98ecb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731522300120 2024-11-13T18:25:05,200 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 279066480573440c9c24af55f3708d1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731522301526 2024-11-13T18:25:05,201 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8eb246bcc94a6c9e09b166ee325e4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731522303757 2024-11-13T18:25:05,235 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#C#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:05,236 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/76f55f36db9d4510a53df2472730c1f0 is 50, key is test_row_0/C:col10/1731522303793/Put/seqid=0 2024-11-13T18:25:05,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:05,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:05,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:05,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:05,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:05,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:05,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742089_1265 (size=12459) 2024-11-13T18:25:05,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/4c0bb97af17c48538cf1332d375a0a87 2024-11-13T18:25:05,339 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/76f55f36db9d4510a53df2472730c1f0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/76f55f36db9d4510a53df2472730c1f0 2024-11-13T18:25:05,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/6bb451e0547b4ef3b606a6e098567f23 is 50, key is test_row_0/C:col10/1731522304693/Put/seqid=0 2024-11-13T18:25:05,363 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/C of 304012adc4150a48d7183d0af2e39078 into 76f55f36db9d4510a53df2472730c1f0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:05,363 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:05,363 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/C, priority=13, startTime=1731522304630; duration=0sec 2024-11-13T18:25:05,363 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:05,363 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:05,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742090_1266 (size=12151) 2024-11-13T18:25:05,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:05,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:05,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:05,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/6bb451e0547b4ef3b606a6e098567f23 2024-11-13T18:25:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/cfb7479b134f41b8abd8c6b7aec76b3f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cfb7479b134f41b8abd8c6b7aec76b3f 2024-11-13T18:25:05,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:05,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522365415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:05,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cfb7479b134f41b8abd8c6b7aec76b3f, entries=200, sequenceid=176, filesize=14.2 K 2024-11-13T18:25:05,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522365416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:05,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522365411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:05,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522365426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:05,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522365435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:05,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/4c0bb97af17c48538cf1332d375a0a87 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4c0bb97af17c48538cf1332d375a0a87 2024-11-13T18:25:05,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4c0bb97af17c48538cf1332d375a0a87, entries=150, sequenceid=176, filesize=11.9 K 2024-11-13T18:25:05,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/6bb451e0547b4ef3b606a6e098567f23 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/6bb451e0547b4ef3b606a6e098567f23 2024-11-13T18:25:05,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/6bb451e0547b4ef3b606a6e098567f23, entries=150, sequenceid=176, filesize=11.9 K 2024-11-13T18:25:05,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 304012adc4150a48d7183d0af2e39078 in 783ms, sequenceid=176, compaction requested=false 2024-11-13T18:25:05,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:05,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:25:05,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:05,568 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:05,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/4c6ccef1315e48d29ec8697dfa5e810f is 50, key is test_row_0/A:col10/1731522304740/Put/seqid=0 2024-11-13T18:25:05,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742091_1267 (size=12151) 2024-11-13T18:25:05,647 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/4c6ccef1315e48d29ec8697dfa5e810f 2024-11-13T18:25:05,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/195f784553f54c9bb2658645124ffcfd is 50, key is test_row_0/B:col10/1731522304740/Put/seqid=0 2024-11-13T18:25:05,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742092_1268 (size=12151) 2024-11-13T18:25:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:25:05,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:06,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522305961, max timestamp=1731522306051Num uniques 859; 2024-11-13T18:25:06,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:06,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522305961, max timestamp=1731522306051Num uniques 859; 2024-11-13T18:25:06,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:06,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522305961, max timestamp=1731522306051Num uniques 859; 2024-11-13T18:25:06,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:25:06,155 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/195f784553f54c9bb2658645124ffcfd 2024-11-13T18:25:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522366143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522366150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731522306099, max timestamp=1731522306160Num uniques 1018; 2024-11-13T18:25:06,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:25:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731522306099, max timestamp=1731522306160Num uniques 1018; 2024-11-13T18:25:06,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:25:06,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731522306099, max timestamp=1731522306160Num uniques 1009; 2024-11-13T18:25:06,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-13T18:25:06,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/91f6ec9ea61b43739de068cc8600b997 is 50, key is test_row_0/C:col10/1731522304740/Put/seqid=0 2024-11-13T18:25:06,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742093_1269 (size=12151) 2024-11-13T18:25:06,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522366281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522366288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731522306251, max timestamp=1731522306296Num uniques 1027; 2024-11-13T18:25:06,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T18:25:06,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731522306251, max timestamp=1731522306296Num uniques 1027; 2024-11-13T18:25:06,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T18:25:06,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731522306251, max timestamp=1731522306296Num uniques 1027; 2024-11-13T18:25:06,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T18:25:06,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522366409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731522306376, max timestamp=1731522306417Num uniques 1036; 2024-11-13T18:25:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731522306376, max timestamp=1731522306417Num uniques 1036; 2024-11-13T18:25:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731522306376, max timestamp=1731522306417Num uniques 1036; 2024-11-13T18:25:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-13T18:25:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-13T18:25:06,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-13T18:25:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522366519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522366528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522366531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522366535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522366538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522366646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,659 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/91f6ec9ea61b43739de068cc8600b997 2024-11-13T18:25:06,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522366658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522366679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/4c6ccef1315e48d29ec8697dfa5e810f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4c6ccef1315e48d29ec8697dfa5e810f 2024-11-13T18:25:06,760 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4c6ccef1315e48d29ec8697dfa5e810f, entries=150, sequenceid=189, filesize=11.9 K 2024-11-13T18:25:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/195f784553f54c9bb2658645124ffcfd as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/195f784553f54c9bb2658645124ffcfd 2024-11-13T18:25:06,772 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/195f784553f54c9bb2658645124ffcfd, entries=150, sequenceid=189, filesize=11.9 K 2024-11-13T18:25:06,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522366769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/91f6ec9ea61b43739de068cc8600b997 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/91f6ec9ea61b43739de068cc8600b997 2024-11-13T18:25:06,805 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/91f6ec9ea61b43739de068cc8600b997, entries=150, sequenceid=189, filesize=11.9 K 2024-11-13T18:25:06,806 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 304012adc4150a48d7183d0af2e39078 in 1238ms, sequenceid=189, compaction requested=true 2024-11-13T18:25:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-13T18:25:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-13T18:25:06,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T18:25:06,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0350 sec 2024-11-13T18:25:06,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 3.0440 sec 2024-11-13T18:25:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:06,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=147.60 KB heapSize=412.92 KB 2024-11-13T18:25:06,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:06,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:06,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5da596225f9f4a7b9c7e512402a16019 is 50, key is test_row_0/A:col10/1731522306501/Put/seqid=0 2024-11-13T18:25:06,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522366868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742094_1270 (size=12301) 2024-11-13T18:25:06,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522366875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522366879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522366898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522366986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522366986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:07,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522367088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:07,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522367196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:07,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522367197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:07,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522367198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:07,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522367218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:07,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5da596225f9f4a7b9c7e512402a16019 2024-11-13T18:25:07,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/d5dbb468950f4f8a953117b81f539074 is 50, key is test_row_0/B:col10/1731522306501/Put/seqid=0 2024-11-13T18:25:07,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742095_1271 (size=12301) 2024-11-13T18:25:07,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/d5dbb468950f4f8a953117b81f539074 2024-11-13T18:25:07,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/ba4e1bdb302a4d78963ae51a71559eb7 is 50, key is test_row_0/C:col10/1731522306501/Put/seqid=0 2024-11-13T18:25:07,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742096_1272 (size=12301) 2024-11-13T18:25:07,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/ba4e1bdb302a4d78963ae51a71559eb7 2024-11-13T18:25:07,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5da596225f9f4a7b9c7e512402a16019 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5da596225f9f4a7b9c7e512402a16019 2024-11-13T18:25:07,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522367506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:07,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522367506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:07,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5da596225f9f4a7b9c7e512402a16019, entries=150, sequenceid=277, filesize=12.0 K 2024-11-13T18:25:07,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/d5dbb468950f4f8a953117b81f539074 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d5dbb468950f4f8a953117b81f539074 2024-11-13T18:25:07,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d5dbb468950f4f8a953117b81f539074, entries=150, sequenceid=277, filesize=12.0 K 2024-11-13T18:25:07,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/ba4e1bdb302a4d78963ae51a71559eb7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ba4e1bdb302a4d78963ae51a71559eb7 2024-11-13T18:25:07,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ba4e1bdb302a4d78963ae51a71559eb7, entries=150, sequenceid=277, filesize=12.0 K 2024-11-13T18:25:07,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~412.88 KB/422784, currentSize=33.54 KB/34350 for 304012adc4150a48d7183d0af2e39078 in 754ms, sequenceid=277, compaction requested=true 2024-11-13T18:25:07,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:07,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:07,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:07,607 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:07,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:07,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:07,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:07,607 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:07,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:07,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:07,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:07,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:07,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:07,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:07,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:07,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:07,627 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:07,627 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51452 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:07,627 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/B is initiating minor compaction (all files) 2024-11-13T18:25:07,627 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/A is initiating minor compaction (all files) 2024-11-13T18:25:07,627 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/B in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:07,627 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/A in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:07,627 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/f39d83cdb1414f4780f57533296aa890, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4c0bb97af17c48538cf1332d375a0a87, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/195f784553f54c9bb2658645124ffcfd, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d5dbb468950f4f8a953117b81f539074] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=47.9 K 2024-11-13T18:25:07,627 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5d67cef49e9a41c6bb0546760f172c98, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cfb7479b134f41b8abd8c6b7aec76b3f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4c6ccef1315e48d29ec8697dfa5e810f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5da596225f9f4a7b9c7e512402a16019] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=50.2 K 2024-11-13T18:25:07,634 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting f39d83cdb1414f4780f57533296aa890, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731522303757 2024-11-13T18:25:07,643 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d67cef49e9a41c6bb0546760f172c98, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731522303757 2024-11-13T18:25:07,643 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c0bb97af17c48538cf1332d375a0a87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731522303974 2024-11-13T18:25:07,647 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfb7479b134f41b8abd8c6b7aec76b3f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731522303974 2024-11-13T18:25:07,647 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 195f784553f54c9bb2658645124ffcfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731522304713 2024-11-13T18:25:07,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/219138a2a88942788d2a46708df3e92f is 50, key is test_row_1/A:col10/1731522307602/Put/seqid=0 2024-11-13T18:25:07,655 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c6ccef1315e48d29ec8697dfa5e810f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731522304713 2024-11-13T18:25:07,655 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting d5dbb468950f4f8a953117b81f539074, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731522306501 2024-11-13T18:25:07,659 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da596225f9f4a7b9c7e512402a16019, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731522306501 2024-11-13T18:25:07,691 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#A#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:07,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742097_1273 (size=12297) 2024-11-13T18:25:07,692 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/9cc60ca98d0b48158666a6ae624d38dd is 50, key is test_row_0/A:col10/1731522306501/Put/seqid=0 2024-11-13T18:25:07,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/219138a2a88942788d2a46708df3e92f 2024-11-13T18:25:07,732 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#B#compaction#227 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:07,732 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/e89bf4ed69b24734b98097f506e111e1 is 50, key is test_row_0/B:col10/1731522306501/Put/seqid=0 2024-11-13T18:25:07,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/84129a1cf5744a9cbcf4adbcf791d179 is 50, key is test_row_1/B:col10/1731522307602/Put/seqid=0 2024-11-13T18:25:07,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742098_1274 (size=12745) 2024-11-13T18:25:07,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742099_1275 (size=12745) 2024-11-13T18:25:07,831 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/e89bf4ed69b24734b98097f506e111e1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e89bf4ed69b24734b98097f506e111e1 2024-11-13T18:25:07,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742100_1276 (size=9857) 2024-11-13T18:25:07,907 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 304012adc4150a48d7183d0af2e39078/B of 304012adc4150a48d7183d0af2e39078 into e89bf4ed69b24734b98097f506e111e1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:07,907 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:07,907 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/B, priority=12, startTime=1731522307607; duration=0sec 2024-11-13T18:25:07,907 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:07,907 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:07,907 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:07,925 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:07,925 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/C is initiating minor compaction (all files) 2024-11-13T18:25:07,926 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/C in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:07,926 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/76f55f36db9d4510a53df2472730c1f0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/6bb451e0547b4ef3b606a6e098567f23, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/91f6ec9ea61b43739de068cc8600b997, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ba4e1bdb302a4d78963ae51a71559eb7] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=47.9 K 2024-11-13T18:25:07,931 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f55f36db9d4510a53df2472730c1f0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731522303757 2024-11-13T18:25:07,939 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb451e0547b4ef3b606a6e098567f23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731522303974 2024-11-13T18:25:07,939 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 91f6ec9ea61b43739de068cc8600b997, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731522304713 2024-11-13T18:25:07,940 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting ba4e1bdb302a4d78963ae51a71559eb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731522306501 2024-11-13T18:25:07,958 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#C#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:07,959 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/b0dd84ae27e346458db0bd507608cd45 is 50, key is test_row_0/C:col10/1731522306501/Put/seqid=0 2024-11-13T18:25:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:25:07,973 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:07,974 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:07,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-13T18:25:07,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522307890, max timestamp=1731522307980Num uniques 859; 2024-11-13T18:25:07,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:07,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522307890, max timestamp=1731522307980Num uniques 859; 2024-11-13T18:25:07,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:07,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522307890, max timestamp=1731522307980Num uniques 859; 2024-11-13T18:25:07,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:25:08,007 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:08,012 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:08,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:08,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742101_1277 (size=12745) 2024-11-13T18:25:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:25:08,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522368096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522368098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731522308064, max timestamp=1731522308110Num uniques 1068; 2024-11-13T18:25:08,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731522308064, max timestamp=1731522308110Num uniques 1068; 2024-11-13T18:25:08,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T18:25:08,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T18:25:08,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731522308064, max timestamp=1731522308110Num uniques 1068; 2024-11-13T18:25:08,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T18:25:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522368100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T18:25:08,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:08,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,227 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/9cc60ca98d0b48158666a6ae624d38dd as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9cc60ca98d0b48158666a6ae624d38dd 2024-11-13T18:25:08,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522308231, max timestamp=1731522308239Num uniques 1027; 2024-11-13T18:25:08,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-11-13T18:25:08,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522308231, max timestamp=1731522308239Num uniques 1027; 2024-11-13T18:25:08,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-11-13T18:25:08,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522308231, max timestamp=1731522308239Num uniques 1027; 2024-11-13T18:25:08,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-11-13T18:25:08,265 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 304012adc4150a48d7183d0af2e39078/A of 304012adc4150a48d7183d0af2e39078 into 9cc60ca98d0b48158666a6ae624d38dd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:08,265 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:08,265 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/A, priority=12, startTime=1731522307607; duration=0sec 2024-11-13T18:25:08,265 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:08,266 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:08,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/84129a1cf5744a9cbcf4adbcf791d179 2024-11-13T18:25:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:25:08,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/ef637fcb124c429b96b02899bf94a53a is 50, key is test_row_1/C:col10/1731522307602/Put/seqid=0 2024-11-13T18:25:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522368343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522368345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T18:25:08,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:08,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522368357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522368359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522368357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742102_1278 (size=9857) 2024-11-13T18:25:08,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522368469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,482 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/b0dd84ae27e346458db0bd507608cd45 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b0dd84ae27e346458db0bd507608cd45 2024-11-13T18:25:08,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522368469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522368476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522368476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522368476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T18:25:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,541 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 304012adc4150a48d7183d0af2e39078/C of 304012adc4150a48d7183d0af2e39078 into b0dd84ae27e346458db0bd507608cd45(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:08,541 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:08,541 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/C, priority=12, startTime=1731522307607; duration=0sec 2024-11-13T18:25:08,541 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:08,541 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:25:08,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T18:25:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522368691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522368696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522368697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522368697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:08,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522368697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:08,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/ef637fcb124c429b96b02899bf94a53a 2024-11-13T18:25:08,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/219138a2a88942788d2a46708df3e92f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/219138a2a88942788d2a46708df3e92f 2024-11-13T18:25:08,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/219138a2a88942788d2a46708df3e92f, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T18:25:08,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/84129a1cf5744a9cbcf4adbcf791d179 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/84129a1cf5744a9cbcf4adbcf791d179 2024-11-13T18:25:08,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/84129a1cf5744a9cbcf4adbcf791d179, entries=100, sequenceid=287, filesize=9.6 K 2024-11-13T18:25:08,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/ef637fcb124c429b96b02899bf94a53a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ef637fcb124c429b96b02899bf94a53a 2024-11-13T18:25:08,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T18:25:08,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:08,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:08,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ef637fcb124c429b96b02899bf94a53a, entries=100, sequenceid=287, filesize=9.6 K 2024-11-13T18:25:08,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=130.57 KB/133704 for 304012adc4150a48d7183d0af2e39078 in 1226ms, sequenceid=287, compaction requested=false 2024-11-13T18:25:08,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:08,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T18:25:08,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:08,998 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=130.57 KB heapSize=375.07 KB 2024-11-13T18:25:08,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:08,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:09,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/242825db7535495c977db04d586bb001 is 50, key is test_row_0/A:col10/1731522308352/Put/seqid=0 2024-11-13T18:25:09,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:09,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742103_1279 (size=12301) 2024-11-13T18:25:09,045 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.52 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/242825db7535495c977db04d586bb001 2024-11-13T18:25:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/bb02e4ca01d845b799b06e765d1795a9 is 50, key is test_row_0/B:col10/1731522308352/Put/seqid=0 2024-11-13T18:25:09,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522369074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522369074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522369092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522369092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522369102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742104_1280 (size=12301) 2024-11-13T18:25:09,112 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.52 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/bb02e4ca01d845b799b06e765d1795a9 2024-11-13T18:25:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:25:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/2d14626423014214a6cf76d9e631754e is 50, key is test_row_0/C:col10/1731522308352/Put/seqid=0 2024-11-13T18:25:09,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742105_1281 (size=12301) 2024-11-13T18:25:09,183 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.52 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/2d14626423014214a6cf76d9e631754e 2024-11-13T18:25:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/242825db7535495c977db04d586bb001 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/242825db7535495c977db04d586bb001 2024-11-13T18:25:09,200 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/242825db7535495c977db04d586bb001, entries=150, sequenceid=361, filesize=12.0 K 2024-11-13T18:25:09,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/bb02e4ca01d845b799b06e765d1795a9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/bb02e4ca01d845b799b06e765d1795a9 2024-11-13T18:25:09,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522369196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,207 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/bb02e4ca01d845b799b06e765d1795a9, entries=150, sequenceid=361, filesize=12.0 K 2024-11-13T18:25:09,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/2d14626423014214a6cf76d9e631754e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2d14626423014214a6cf76d9e631754e 2024-11-13T18:25:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522369205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,215 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2d14626423014214a6cf76d9e631754e, entries=150, sequenceid=361, filesize=12.0 K 2024-11-13T18:25:09,225 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~130.57 KB/133704, heapSize ~375.02 KB/384024, currentSize=53.67 KB/54960 for 304012adc4150a48d7183d0af2e39078 in 227ms, sequenceid=361, compaction requested=true 2024-11-13T18:25:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-13T18:25:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-13T18:25:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:09,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:25:09,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-13T18:25:09,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2190 sec 2024-11-13T18:25:09,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.2620 sec 2024-11-13T18:25:09,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:09,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:09,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:09,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:09,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:09,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:09,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/7f97ad35737f48ddaef94535d3a5ccb9 is 50, key is test_row_0/A:col10/1731522309032/Put/seqid=0 2024-11-13T18:25:09,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742106_1282 (size=14741) 2024-11-13T18:25:09,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/7f97ad35737f48ddaef94535d3a5ccb9 2024-11-13T18:25:09,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522369318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522369327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522369331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/08dbc8e994304fd3a0b2338263125586 is 50, key is test_row_0/B:col10/1731522309032/Put/seqid=0 2024-11-13T18:25:09,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742107_1283 (size=12301) 2024-11-13T18:25:09,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522369416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522369416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522369439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522369446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522369446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522369656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522369665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522369671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,694 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:25:09,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522369726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522369727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/08dbc8e994304fd3a0b2338263125586 2024-11-13T18:25:09,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522369966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522369975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:09,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522369985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:10,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/2009e175c48c4cc6a3ceef44a64d2483 is 50, key is test_row_0/C:col10/1731522309032/Put/seqid=0 2024-11-13T18:25:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:25:10,136 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:10,136 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-13T18:25:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:25:10,140 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:10,141 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:10,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:10,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742108_1284 (size=12301) 2024-11-13T18:25:10,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522370239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:10,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:10,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:10,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:25:10,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522370245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:10,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T18:25:10,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:10,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:10,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:10,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:10,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:10,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T18:25:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:25:10,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:10,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:10,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:10,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:10,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:10,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522370476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:10,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522370486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:10,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:10,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:10,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:10,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522370496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:10,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/2009e175c48c4cc6a3ceef44a64d2483 2024-11-13T18:25:10,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/7f97ad35737f48ddaef94535d3a5ccb9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/7f97ad35737f48ddaef94535d3a5ccb9 2024-11-13T18:25:10,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T18:25:10,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:10,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:10,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/7f97ad35737f48ddaef94535d3a5ccb9, entries=200, sequenceid=374, filesize=14.4 K 2024-11-13T18:25:10,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/08dbc8e994304fd3a0b2338263125586 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/08dbc8e994304fd3a0b2338263125586 2024-11-13T18:25:10,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/08dbc8e994304fd3a0b2338263125586, entries=150, sequenceid=374, filesize=12.0 K 2024-11-13T18:25:10,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/2009e175c48c4cc6a3ceef44a64d2483 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2009e175c48c4cc6a3ceef44a64d2483 2024-11-13T18:25:10,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2009e175c48c4cc6a3ceef44a64d2483, entries=150, sequenceid=374, filesize=12.0 K 2024-11-13T18:25:10,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 304012adc4150a48d7183d0af2e39078 in 1420ms, sequenceid=374, compaction requested=true 2024-11-13T18:25:10,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:10,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:10,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:10,652 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:10,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:10,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:10,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:10,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:10,653 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:10,653 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/A is initiating minor compaction (all files) 2024-11-13T18:25:10,653 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/A in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:10,653 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9cc60ca98d0b48158666a6ae624d38dd, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/219138a2a88942788d2a46708df3e92f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/242825db7535495c977db04d586bb001, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/7f97ad35737f48ddaef94535d3a5ccb9] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=50.9 K 2024-11-13T18:25:10,654 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:10,656 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc60ca98d0b48158666a6ae624d38dd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731522306501 2024-11-13T18:25:10,657 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 219138a2a88942788d2a46708df3e92f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731522306867 2024-11-13T18:25:10,657 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 242825db7535495c977db04d586bb001, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731522308335 2024-11-13T18:25:10,658 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f97ad35737f48ddaef94535d3a5ccb9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731522309032 2024-11-13T18:25:10,659 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47204 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:10,659 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/B is initiating minor compaction (all files) 2024-11-13T18:25:10,659 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/B in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:10,659 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e89bf4ed69b24734b98097f506e111e1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/84129a1cf5744a9cbcf4adbcf791d179, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/bb02e4ca01d845b799b06e765d1795a9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/08dbc8e994304fd3a0b2338263125586] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=46.1 K 2024-11-13T18:25:10,659 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e89bf4ed69b24734b98097f506e111e1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731522306501 2024-11-13T18:25:10,660 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 84129a1cf5744a9cbcf4adbcf791d179, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731522307602 2024-11-13T18:25:10,662 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting bb02e4ca01d845b799b06e765d1795a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731522308335 2024-11-13T18:25:10,662 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 08dbc8e994304fd3a0b2338263125586, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731522309032 2024-11-13T18:25:10,681 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:10,682 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#B#compaction#238 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:10,682 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/259a6ec85d7d462ab90bd0a5f3f61a26 is 50, key is test_row_0/A:col10/1731522309032/Put/seqid=0 2024-11-13T18:25:10,684 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/00b50a76f23f4ea087a290f3a2527cb4 is 50, key is test_row_0/B:col10/1731522309032/Put/seqid=0 2024-11-13T18:25:10,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742110_1286 (size=12881) 2024-11-13T18:25:10,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742109_1285 (size=12881) 2024-11-13T18:25:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:25:10,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T18:25:10,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:10,776 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:25:10,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:10,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:10,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:10,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:10,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:10,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:10,780 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/259a6ec85d7d462ab90bd0a5f3f61a26 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/259a6ec85d7d462ab90bd0a5f3f61a26 2024-11-13T18:25:10,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/15fde0ed03a54db8bf64af23bc2bee8f is 50, key is test_row_0/A:col10/1731522309325/Put/seqid=0 2024-11-13T18:25:10,791 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 304012adc4150a48d7183d0af2e39078/A of 304012adc4150a48d7183d0af2e39078 into 259a6ec85d7d462ab90bd0a5f3f61a26(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:10,791 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:10,794 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/A, priority=12, startTime=1731522310651; duration=0sec 2024-11-13T18:25:10,795 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:10,795 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:10,795 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:10,797 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47204 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:10,798 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/C is initiating minor compaction (all files) 2024-11-13T18:25:10,798 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/C in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:10,798 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b0dd84ae27e346458db0bd507608cd45, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ef637fcb124c429b96b02899bf94a53a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2d14626423014214a6cf76d9e631754e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2009e175c48c4cc6a3ceef44a64d2483] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=46.1 K 2024-11-13T18:25:10,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742111_1287 (size=12301) 2024-11-13T18:25:10,802 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/15fde0ed03a54db8bf64af23bc2bee8f 2024-11-13T18:25:10,803 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0dd84ae27e346458db0bd507608cd45, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731522306501 2024-11-13T18:25:10,805 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef637fcb124c429b96b02899bf94a53a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731522307602 2024-11-13T18:25:10,806 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d14626423014214a6cf76d9e631754e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731522308335 2024-11-13T18:25:10,806 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2009e175c48c4cc6a3ceef44a64d2483, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731522309032 2024-11-13T18:25:10,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a6bb87a5fe48400daaf54b50f80c9d1a is 50, key is test_row_0/B:col10/1731522309325/Put/seqid=0 2024-11-13T18:25:10,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742112_1288 (size=12301) 2024-11-13T18:25:10,864 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a6bb87a5fe48400daaf54b50f80c9d1a 2024-11-13T18:25:10,872 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#C#compaction#241 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:10,873 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/c9ef1db1030a41a2bf9b7e18341c6769 is 50, key is test_row_0/C:col10/1731522309032/Put/seqid=0 2024-11-13T18:25:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/4b07d03625ec4681a3d81bd03ab662a0 is 50, key is test_row_0/C:col10/1731522309325/Put/seqid=0 2024-11-13T18:25:10,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742113_1289 (size=12881) 2024-11-13T18:25:10,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742114_1290 (size=12301) 2024-11-13T18:25:10,990 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/4b07d03625ec4681a3d81bd03ab662a0 2024-11-13T18:25:11,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/15fde0ed03a54db8bf64af23bc2bee8f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/15fde0ed03a54db8bf64af23bc2bee8f 2024-11-13T18:25:11,039 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/15fde0ed03a54db8bf64af23bc2bee8f, entries=150, sequenceid=393, filesize=12.0 K 2024-11-13T18:25:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a6bb87a5fe48400daaf54b50f80c9d1a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a6bb87a5fe48400daaf54b50f80c9d1a 2024-11-13T18:25:11,063 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a6bb87a5fe48400daaf54b50f80c9d1a, entries=150, sequenceid=393, filesize=12.0 K 2024-11-13T18:25:11,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/4b07d03625ec4681a3d81bd03ab662a0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b07d03625ec4681a3d81bd03ab662a0 2024-11-13T18:25:11,079 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b07d03625ec4681a3d81bd03ab662a0, entries=150, sequenceid=393, filesize=12.0 K 2024-11-13T18:25:11,080 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 304012adc4150a48d7183d0af2e39078 in 304ms, sequenceid=393, compaction requested=false 2024-11-13T18:25:11,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:11,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:11,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-13T18:25:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-13T18:25:11,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-13T18:25:11,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 942 msec 2024-11-13T18:25:11,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 951 msec 2024-11-13T18:25:11,174 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/00b50a76f23f4ea087a290f3a2527cb4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/00b50a76f23f4ea087a290f3a2527cb4 2024-11-13T18:25:11,180 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 304012adc4150a48d7183d0af2e39078/B of 304012adc4150a48d7183d0af2e39078 into 00b50a76f23f4ea087a290f3a2527cb4(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:11,180 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:11,180 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/B, priority=12, startTime=1731522310652; duration=0sec 2024-11-13T18:25:11,181 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:11,181 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:11,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:25:11,275 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:11,276 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-13T18:25:11,278 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:25:11,280 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:11,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:11,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/d7f097ad8a924d75946a2bf144d4d1cd is 50, key is test_row_0/A:col10/1731522311270/Put/seqid=0 2024-11-13T18:25:11,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742115_1291 (size=9857) 2024-11-13T18:25:11,330 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/c9ef1db1030a41a2bf9b7e18341c6769 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/c9ef1db1030a41a2bf9b7e18341c6769 2024-11-13T18:25:11,341 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 304012adc4150a48d7183d0af2e39078/C of 304012adc4150a48d7183d0af2e39078 into c9ef1db1030a41a2bf9b7e18341c6769(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:11,341 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:11,341 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/C, priority=12, startTime=1731522310652; duration=0sec 2024-11-13T18:25:11,341 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:11,341 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:11,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522311342, max timestamp=1731522311359Num uniques 859; 2024-11-13T18:25:11,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522311342, max timestamp=1731522311359Num uniques 859; 2024-11-13T18:25:11,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:11,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522311342, max timestamp=1731522311359Num uniques 859; 2024-11-13T18:25:11,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:11,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:25:11,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:11,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522371417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522371417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:11,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:11,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:11,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T18:25:11,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:11,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:11,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:11,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522371497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:11,502 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in 304012adc4150a48d7183d0af2e39078 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:25:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522371505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:11,509 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_1' in 304012adc4150a48d7183d0af2e39078 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522371505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:11,511 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in 304012adc4150a48d7183d0af2e39078 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:25:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:11,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522371537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:11,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522371539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:11,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:11,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:25:11,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T18:25:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:11,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:11,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/d7f097ad8a924d75946a2bf144d4d1cd 2024-11-13T18:25:11,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/9acb344936ac497082334ab3a146f862 is 50, key is test_row_0/B:col10/1731522311270/Put/seqid=0 2024-11-13T18:25:11,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522371746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522371747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:11,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T18:25:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:11,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:11,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:11,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742116_1292 (size=9857) 2024-11-13T18:25:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:25:11,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T18:25:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:11,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:12,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522372056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:12,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:12,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522372056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:12,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:12,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T18:25:12,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:12,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:12,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/9acb344936ac497082334ab3a146f862 2024-11-13T18:25:12,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/58166833ec1b4b2ba99efca80eb6663b is 50, key is test_row_0/C:col10/1731522311270/Put/seqid=0 2024-11-13T18:25:12,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T18:25:12,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:12,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:12,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:12,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:12,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:12,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742117_1293 (size=9857) 2024-11-13T18:25:12,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/58166833ec1b4b2ba99efca80eb6663b 2024-11-13T18:25:12,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/d7f097ad8a924d75946a2bf144d4d1cd as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d7f097ad8a924d75946a2bf144d4d1cd 2024-11-13T18:25:12,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d7f097ad8a924d75946a2bf144d4d1cd, entries=100, sequenceid=405, filesize=9.6 K 2024-11-13T18:25:12,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/9acb344936ac497082334ab3a146f862 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/9acb344936ac497082334ab3a146f862 2024-11-13T18:25:12,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/9acb344936ac497082334ab3a146f862, entries=100, sequenceid=405, filesize=9.6 K 2024-11-13T18:25:12,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/58166833ec1b4b2ba99efca80eb6663b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/58166833ec1b4b2ba99efca80eb6663b 2024-11-13T18:25:12,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/58166833ec1b4b2ba99efca80eb6663b, entries=100, sequenceid=405, filesize=9.6 K 2024-11-13T18:25:12,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 304012adc4150a48d7183d0af2e39078 in 1090ms, sequenceid=405, compaction requested=true 2024-11-13T18:25:12,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:25:12,364 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:12,364 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:12,374 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:12,374 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/A is initiating minor compaction (all files) 2024-11-13T18:25:12,374 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/A in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:12,374 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/259a6ec85d7d462ab90bd0a5f3f61a26, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/15fde0ed03a54db8bf64af23bc2bee8f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d7f097ad8a924d75946a2bf144d4d1cd] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=34.2 K 2024-11-13T18:25:12,375 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 259a6ec85d7d462ab90bd0a5f3f61a26, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731522309032 2024-11-13T18:25:12,375 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 15fde0ed03a54db8bf64af23bc2bee8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731522309311 2024-11-13T18:25:12,379 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:12,379 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/C is initiating minor compaction (all files) 2024-11-13T18:25:12,379 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/C in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:12,379 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f097ad8a924d75946a2bf144d4d1cd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731522311270 2024-11-13T18:25:12,379 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/c9ef1db1030a41a2bf9b7e18341c6769, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b07d03625ec4681a3d81bd03ab662a0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/58166833ec1b4b2ba99efca80eb6663b] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=34.2 K 2024-11-13T18:25:12,383 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9ef1db1030a41a2bf9b7e18341c6769, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731522309032 2024-11-13T18:25:12,383 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b07d03625ec4681a3d81bd03ab662a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731522309311 2024-11-13T18:25:12,395 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58166833ec1b4b2ba99efca80eb6663b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731522311270 2024-11-13T18:25:12,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T18:25:12,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:12,415 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T18:25:12,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:25:12,436 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:12,437 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5894221438924d39b8d3eb38e55c8f89 is 50, key is test_row_0/A:col10/1731522311270/Put/seqid=0 2024-11-13T18:25:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/94b6010e4a2a43778f9b52dbd50c675c is 50, key is test_row_0/A:col10/1731522311414/Put/seqid=0 2024-11-13T18:25:12,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742119_1295 (size=12301) 2024-11-13T18:25:12,486 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/94b6010e4a2a43778f9b52dbd50c675c 2024-11-13T18:25:12,488 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#C#compaction#248 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:12,489 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/cc5894e83f0647049ffc0bc6c8490632 is 50, key is test_row_0/C:col10/1731522311270/Put/seqid=0 2024-11-13T18:25:12,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742118_1294 (size=12983) 2024-11-13T18:25:12,517 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5894221438924d39b8d3eb38e55c8f89 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5894221438924d39b8d3eb38e55c8f89 2024-11-13T18:25:12,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/740a314ef6c944a4b23787ab336969fe is 50, key is test_row_0/B:col10/1731522311414/Put/seqid=0 2024-11-13T18:25:12,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742120_1296 (size=12983) 2024-11-13T18:25:12,544 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/A of 304012adc4150a48d7183d0af2e39078 into 5894221438924d39b8d3eb38e55c8f89(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:12,544 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:12,544 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/A, priority=13, startTime=1731522312363; duration=0sec 2024-11-13T18:25:12,544 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:12,545 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:12,545 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:12,555 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:12,555 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/B is initiating minor compaction (all files) 2024-11-13T18:25:12,555 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/B in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:12,555 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/00b50a76f23f4ea087a290f3a2527cb4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a6bb87a5fe48400daaf54b50f80c9d1a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/9acb344936ac497082334ab3a146f862] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=34.2 K 2024-11-13T18:25:12,556 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b50a76f23f4ea087a290f3a2527cb4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731522309032 2024-11-13T18:25:12,556 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting a6bb87a5fe48400daaf54b50f80c9d1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731522309311 2024-11-13T18:25:12,557 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9acb344936ac497082334ab3a146f862, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731522311270 2024-11-13T18:25:12,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742121_1297 (size=12301) 2024-11-13T18:25:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:12,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:12,575 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/740a314ef6c944a4b23787ab336969fe 2024-11-13T18:25:12,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/714514cb34c74028aa2a711c7be53203 is 50, key is test_row_0/C:col10/1731522311414/Put/seqid=0 2024-11-13T18:25:12,593 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#B#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:12,594 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/0e37f5727b1a48a9a854cd50ced15f7c is 50, key is test_row_0/B:col10/1731522311270/Put/seqid=0 2024-11-13T18:25:12,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522372605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:12,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522372606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:12,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742122_1298 (size=12301) 2024-11-13T18:25:12,643 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/714514cb34c74028aa2a711c7be53203 2024-11-13T18:25:12,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/94b6010e4a2a43778f9b52dbd50c675c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/94b6010e4a2a43778f9b52dbd50c675c 2024-11-13T18:25:12,659 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/94b6010e4a2a43778f9b52dbd50c675c, entries=150, sequenceid=443, filesize=12.0 K 2024-11-13T18:25:12,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/740a314ef6c944a4b23787ab336969fe as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/740a314ef6c944a4b23787ab336969fe 2024-11-13T18:25:12,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742123_1299 (size=12983) 2024-11-13T18:25:12,670 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/740a314ef6c944a4b23787ab336969fe, entries=150, sequenceid=443, filesize=12.0 K 2024-11-13T18:25:12,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/714514cb34c74028aa2a711c7be53203 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/714514cb34c74028aa2a711c7be53203 2024-11-13T18:25:12,680 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/714514cb34c74028aa2a711c7be53203, entries=150, sequenceid=443, filesize=12.0 K 2024-11-13T18:25:12,681 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 304012adc4150a48d7183d0af2e39078 in 266ms, sequenceid=443, compaction requested=false 2024-11-13T18:25:12,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:12,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:12,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-13T18:25:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-13T18:25:12,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-13T18:25:12,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-13T18:25:12,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4120 sec 2024-11-13T18:25:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:12,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:25:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:12,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5952178bd3464a589542f0c4afb63057 is 50, key is test_row_0/A:col10/1731522312604/Put/seqid=0 2024-11-13T18:25:12,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742124_1300 (size=14741) 2024-11-13T18:25:12,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5952178bd3464a589542f0c4afb63057 2024-11-13T18:25:12,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/04e4e05a7dac42d6aaa16473f17af839 is 50, key is test_row_0/B:col10/1731522312604/Put/seqid=0 2024-11-13T18:25:12,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522312804, max timestamp=1731522312833Num uniques 859; 2024-11-13T18:25:12,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:12,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522312804, max timestamp=1731522312833Num uniques 859; 2024-11-13T18:25:12,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:12,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522312804, max timestamp=1731522312833Num uniques 859; 2024-11-13T18:25:12,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:12,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742125_1301 (size=12301) 2024-11-13T18:25:12,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/04e4e05a7dac42d6aaa16473f17af839 2024-11-13T18:25:12,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/061991408bed4f27bf4032feb2ed599e is 50, key is test_row_0/C:col10/1731522312604/Put/seqid=0 2024-11-13T18:25:12,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742126_1302 (size=12301) 2024-11-13T18:25:12,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/061991408bed4f27bf4032feb2ed599e 2024-11-13T18:25:12,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5952178bd3464a589542f0c4afb63057 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5952178bd3464a589542f0c4afb63057 2024-11-13T18:25:12,947 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/cc5894e83f0647049ffc0bc6c8490632 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/cc5894e83f0647049ffc0bc6c8490632 2024-11-13T18:25:12,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5952178bd3464a589542f0c4afb63057, entries=200, sequenceid=455, filesize=14.4 K 2024-11-13T18:25:12,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522372944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522372945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:12,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/04e4e05a7dac42d6aaa16473f17af839 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04e4e05a7dac42d6aaa16473f17af839 2024-11-13T18:25:12,963 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/C of 304012adc4150a48d7183d0af2e39078 into cc5894e83f0647049ffc0bc6c8490632(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:12,963 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:12,963 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/C, priority=13, startTime=1731522312364; duration=0sec 2024-11-13T18:25:12,963 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:12,963 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:12,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04e4e05a7dac42d6aaa16473f17af839, entries=150, sequenceid=455, filesize=12.0 K 2024-11-13T18:25:12,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/061991408bed4f27bf4032feb2ed599e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/061991408bed4f27bf4032feb2ed599e 2024-11-13T18:25:12,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/061991408bed4f27bf4032feb2ed599e, entries=150, sequenceid=455, filesize=12.0 K 2024-11-13T18:25:12,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 304012adc4150a48d7183d0af2e39078 in 253ms, sequenceid=455, compaction requested=true 2024-11-13T18:25:12,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:12,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:12,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:12,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:12,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:12,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:12,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:25:12,978 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-13T18:25:12,979 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:25:12,979 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:25:12,980 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. because compaction request was cancelled 2024-11-13T18:25:12,980 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:12,980 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:12,981 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:12,981 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/C is initiating minor compaction (all files) 2024-11-13T18:25:12,981 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/C in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:12,981 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/cc5894e83f0647049ffc0bc6c8490632, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/714514cb34c74028aa2a711c7be53203, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/061991408bed4f27bf4032feb2ed599e] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=36.7 K 2024-11-13T18:25:12,981 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5894e83f0647049ffc0bc6c8490632, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731522309311 2024-11-13T18:25:12,983 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 714514cb34c74028aa2a711c7be53203, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731522311408 2024-11-13T18:25:12,983 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 061991408bed4f27bf4032feb2ed599e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731522312602 2024-11-13T18:25:13,025 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:13,026 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/9426f60a1a24415caf2e52b2333520a0 is 50, key is test_row_0/C:col10/1731522312604/Put/seqid=0 2024-11-13T18:25:13,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742127_1303 (size=13085) 2024-11-13T18:25:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:13,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-11-13T18:25:13,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:13,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:25:13,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:13,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:25:13,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:13,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:25:13,086 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/0e37f5727b1a48a9a854cd50ced15f7c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/0e37f5727b1a48a9a854cd50ced15f7c 2024-11-13T18:25:13,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/cb2f8a09225845dcb45cef09ac8675d1 is 50, key is test_row_0/A:col10/1731522312918/Put/seqid=0 2024-11-13T18:25:13,102 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/B of 304012adc4150a48d7183d0af2e39078 into 0e37f5727b1a48a9a854cd50ced15f7c(size=12.7 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:13,102 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:13,103 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/B, priority=13, startTime=1731522312364; duration=0sec 2024-11-13T18:25:13,104 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:13,104 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:13,105 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:13,109 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:13,109 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/A is initiating minor compaction (all files) 2024-11-13T18:25:13,109 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/A in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:13,109 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5894221438924d39b8d3eb38e55c8f89, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/94b6010e4a2a43778f9b52dbd50c675c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5952178bd3464a589542f0c4afb63057] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=39.1 K 2024-11-13T18:25:13,110 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 5894221438924d39b8d3eb38e55c8f89, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731522309311 2024-11-13T18:25:13,111 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b6010e4a2a43778f9b52dbd50c675c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731522311408 2024-11-13T18:25:13,112 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 5952178bd3464a589542f0c4afb63057, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731522312576 2024-11-13T18:25:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522373104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522373109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742128_1304 (size=14741) 2024-11-13T18:25:13,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/cb2f8a09225845dcb45cef09ac8675d1 2024-11-13T18:25:13,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/ba544a0ae11a4f0ebda41a561ea29575 is 50, key is test_row_0/B:col10/1731522312918/Put/seqid=0 2024-11-13T18:25:13,206 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:13,206 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/9a4ecf65d16748ce9fce2b06cafee3fd is 50, key is test_row_0/A:col10/1731522312604/Put/seqid=0 2024-11-13T18:25:13,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522373227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522373227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742129_1305 (size=12301) 2024-11-13T18:25:13,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/ba544a0ae11a4f0ebda41a561ea29575 2024-11-13T18:25:13,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742130_1306 (size=13085) 2024-11-13T18:25:13,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/8da99c76217049a2828309195667a538 is 50, key is test_row_0/C:col10/1731522312918/Put/seqid=0 2024-11-13T18:25:13,322 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/9a4ecf65d16748ce9fce2b06cafee3fd as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a4ecf65d16748ce9fce2b06cafee3fd 2024-11-13T18:25:13,330 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/A of 304012adc4150a48d7183d0af2e39078 into 9a4ecf65d16748ce9fce2b06cafee3fd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:13,330 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:13,330 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/A, priority=13, startTime=1731522312977; duration=0sec 2024-11-13T18:25:13,330 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:13,331 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:13,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742131_1307 (size=12301) 2024-11-13T18:25:13,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/8da99c76217049a2828309195667a538 2024-11-13T18:25:13,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/cb2f8a09225845dcb45cef09ac8675d1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cb2f8a09225845dcb45cef09ac8675d1 2024-11-13T18:25:13,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cb2f8a09225845dcb45cef09ac8675d1, entries=200, sequenceid=494, filesize=14.4 K 2024-11-13T18:25:13,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/ba544a0ae11a4f0ebda41a561ea29575 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ba544a0ae11a4f0ebda41a561ea29575 2024-11-13T18:25:13,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ba544a0ae11a4f0ebda41a561ea29575, entries=150, sequenceid=494, filesize=12.0 K 2024-11-13T18:25:13,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/8da99c76217049a2828309195667a538 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8da99c76217049a2828309195667a538 2024-11-13T18:25:13,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8da99c76217049a2828309195667a538, entries=150, sequenceid=494, filesize=12.0 K 2024-11-13T18:25:13,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~415.72 KB/425696, currentSize=33.54 KB/34350 for 304012adc4150a48d7183d0af2e39078 in 344ms, sequenceid=494, compaction requested=true 2024-11-13T18:25:13,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:13,416 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:25:13,418 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:25:13,418 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:25:13,418 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. because compaction request was cancelled 2024-11-13T18:25:13,419 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:13,419 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:13,421 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:13,421 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/B is initiating minor compaction (all files) 2024-11-13T18:25:13,421 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/B in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:13,421 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/0e37f5727b1a48a9a854cd50ced15f7c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/740a314ef6c944a4b23787ab336969fe, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04e4e05a7dac42d6aaa16473f17af839, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ba544a0ae11a4f0ebda41a561ea29575] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=48.7 K 2024-11-13T18:25:13,422 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e37f5727b1a48a9a854cd50ced15f7c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731522309311 2024-11-13T18:25:13,422 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 740a314ef6c944a4b23787ab336969fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731522311408 2024-11-13T18:25:13,422 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e4e05a7dac42d6aaa16473f17af839, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731522312602 2024-11-13T18:25:13,423 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting ba544a0ae11a4f0ebda41a561ea29575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731522312918 2024-11-13T18:25:13,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:13,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:13,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:13,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:13,442 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#B#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:13,443 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/f00d174780a34dc7a9195524e426f873 is 50, key is test_row_0/B:col10/1731522312918/Put/seqid=0 2024-11-13T18:25:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:25:13,448 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:13,451 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:13,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:13,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:13,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:13,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:13,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:13,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:13,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-13T18:25:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:25:13,463 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:13,464 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:13,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:13,475 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/9426f60a1a24415caf2e52b2333520a0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9426f60a1a24415caf2e52b2333520a0 2024-11-13T18:25:13,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5aae966e70b0447ab74bd95040f37676 is 50, key is test_row_0/A:col10/1731522313076/Put/seqid=0 2024-11-13T18:25:13,505 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/C of 304012adc4150a48d7183d0af2e39078 into 9426f60a1a24415caf2e52b2333520a0(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:13,505 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:13,505 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/C, priority=13, startTime=1731522312978; duration=0sec 2024-11-13T18:25:13,506 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:13,506 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:13,506 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:13,506 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:25:13,507 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:25:13,507 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:25:13,507 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. because compaction request was cancelled 2024-11-13T18:25:13,507 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:13,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742132_1308 (size=13119) 2024-11-13T18:25:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:25:13,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742133_1309 (size=17181) 2024-11-13T18:25:13,607 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/f00d174780a34dc7a9195524e426f873 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/f00d174780a34dc7a9195524e426f873 2024-11-13T18:25:13,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5aae966e70b0447ab74bd95040f37676 2024-11-13T18:25:13,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522313551, max timestamp=1731522313613Num uniques 859; 2024-11-13T18:25:13,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:13,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522313551, max timestamp=1731522313613Num uniques 859; 2024-11-13T18:25:13,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:13,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:25:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:13,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:13,643 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 304012adc4150a48d7183d0af2e39078/B of 304012adc4150a48d7183d0af2e39078 into f00d174780a34dc7a9195524e426f873(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:13,643 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:13,643 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/B, priority=12, startTime=1731522313419; duration=0sec 2024-11-13T18:25:13,643 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:13,643 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:13,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/035b532fa9904907960ab5242ca879a3 is 50, key is test_row_0/B:col10/1731522313076/Put/seqid=0 2024-11-13T18:25:13,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522373646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522373649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522373651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522373649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522373679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742134_1310 (size=12301) 2024-11-13T18:25:13,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/035b532fa9904907960ab5242ca879a3 2024-11-13T18:25:13,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/014bf73db5bf452f929b1fac5fdcf66d is 50, key is test_row_0/C:col10/1731522313076/Put/seqid=0 2024-11-13T18:25:13,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:25:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:13,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:13,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522373786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522373796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522373799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:25:13,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522373807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522373808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:13,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742135_1311 (size=12301) 2024-11-13T18:25:13,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:25:13,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:13,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:13,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:13,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:13,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522374007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522374008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522374018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522374024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522374028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:25:14,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:25:14,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/014bf73db5bf452f929b1fac5fdcf66d 2024-11-13T18:25:14,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/5aae966e70b0447ab74bd95040f37676 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5aae966e70b0447ab74bd95040f37676 2024-11-13T18:25:14,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5aae966e70b0447ab74bd95040f37676, entries=250, sequenceid=506, filesize=16.8 K 2024-11-13T18:25:14,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/035b532fa9904907960ab5242ca879a3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/035b532fa9904907960ab5242ca879a3 2024-11-13T18:25:14,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/035b532fa9904907960ab5242ca879a3, entries=150, sequenceid=506, filesize=12.0 K 2024-11-13T18:25:14,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/014bf73db5bf452f929b1fac5fdcf66d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/014bf73db5bf452f929b1fac5fdcf66d 2024-11-13T18:25:14,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:25:14,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:14,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/014bf73db5bf452f929b1fac5fdcf66d, entries=150, sequenceid=506, filesize=12.0 K 2024-11-13T18:25:14,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 304012adc4150a48d7183d0af2e39078 in 833ms, sequenceid=506, compaction requested=true 2024-11-13T18:25:14,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:14,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:14,291 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:14,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:14,291 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:25:14,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:14,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:14,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:14,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:14,293 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:25:14,294 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:25:14,294 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. because compaction request was cancelled 2024-11-13T18:25:14,294 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:14,294 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:14,294 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:14,294 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/A is initiating minor compaction (all files) 2024-11-13T18:25:14,294 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/A in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,294 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a4ecf65d16748ce9fce2b06cafee3fd, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cb2f8a09225845dcb45cef09ac8675d1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5aae966e70b0447ab74bd95040f37676] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=44.0 K 2024-11-13T18:25:14,295 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a4ecf65d16748ce9fce2b06cafee3fd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731522312602 2024-11-13T18:25:14,295 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:14,295 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/C is initiating minor compaction (all files) 2024-11-13T18:25:14,295 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/C in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,295 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9426f60a1a24415caf2e52b2333520a0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8da99c76217049a2828309195667a538, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/014bf73db5bf452f929b1fac5fdcf66d] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=36.8 K 2024-11-13T18:25:14,296 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb2f8a09225845dcb45cef09ac8675d1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731522312918 2024-11-13T18:25:14,296 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9426f60a1a24415caf2e52b2333520a0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731522312602 2024-11-13T18:25:14,296 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aae966e70b0447ab74bd95040f37676, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731522313076 2024-11-13T18:25:14,296 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 8da99c76217049a2828309195667a538, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731522312918 2024-11-13T18:25:14,297 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 014bf73db5bf452f929b1fac5fdcf66d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731522313076 2024-11-13T18:25:14,315 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:14,316 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/0aaf711012c5414eb96750a6f690b730 is 50, key is test_row_0/A:col10/1731522313076/Put/seqid=0 2024-11-13T18:25:14,321 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#C#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:14,321 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/305b68ca62c945f780e43d62492de916 is 50, key is test_row_0/C:col10/1731522313076/Put/seqid=0 2024-11-13T18:25:14,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=138.65 KB heapSize=403.23 KB 2024-11-13T18:25:14,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:14,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:14,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:14,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:14,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:14,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:14,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/68441d53fe7d4719839a4c564e8adca6 is 50, key is test_row_0/A:col10/1731522313648/Put/seqid=0 2024-11-13T18:25:14,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522374357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522374362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522374364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522374357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522374371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742136_1312 (size=13187) 2024-11-13T18:25:14,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742137_1313 (size=13187) 2024-11-13T18:25:14,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:25:14,435 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/305b68ca62c945f780e43d62492de916 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/305b68ca62c945f780e43d62492de916 2024-11-13T18:25:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742138_1314 (size=14741) 2024-11-13T18:25:14,446 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/C of 304012adc4150a48d7183d0af2e39078 into 305b68ca62c945f780e43d62492de916(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:14,446 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:14,446 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/C, priority=13, startTime=1731522314291; duration=0sec 2024-11-13T18:25:14,446 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:14,446 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:14,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/68441d53fe7d4719839a4c564e8adca6 2024-11-13T18:25:14,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/e10579f251f44996b67a0e9896c0ae44 is 50, key is test_row_0/B:col10/1731522313648/Put/seqid=0 2024-11-13T18:25:14,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522374490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522374491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522374490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522374491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522374491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742139_1315 (size=12301) 2024-11-13T18:25:14,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:25:14,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:14,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:25:14,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522374717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522374718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522374718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522374720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522374720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:14,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:25:14,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:14,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,855 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/0aaf711012c5414eb96750a6f690b730 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/0aaf711012c5414eb96750a6f690b730 2024-11-13T18:25:14,899 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/A of 304012adc4150a48d7183d0af2e39078 into 0aaf711012c5414eb96750a6f690b730(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:14,899 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:14,899 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/A, priority=13, startTime=1731522314290; duration=0sec 2024-11-13T18:25:14,899 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:14,899 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:14,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:25:14,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:14,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:14,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:14,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/e10579f251f44996b67a0e9896c0ae44 2024-11-13T18:25:14,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/a3fa055932cc46c6a07b72481d9e2970 is 50, key is test_row_0/C:col10/1731522313648/Put/seqid=0 2024-11-13T18:25:15,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742140_1316 (size=12301) 2024-11-13T18:25:15,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/a3fa055932cc46c6a07b72481d9e2970 2024-11-13T18:25:15,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/68441d53fe7d4719839a4c564e8adca6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/68441d53fe7d4719839a4c564e8adca6 2024-11-13T18:25:15,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/68441d53fe7d4719839a4c564e8adca6, entries=200, sequenceid=541, filesize=14.4 K 2024-11-13T18:25:15,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/e10579f251f44996b67a0e9896c0ae44 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e10579f251f44996b67a0e9896c0ae44 2024-11-13T18:25:15,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522375036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522375036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522375038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e10579f251f44996b67a0e9896c0ae44, entries=150, sequenceid=541, filesize=12.0 K 2024-11-13T18:25:15,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522375046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/a3fa055932cc46c6a07b72481d9e2970 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/a3fa055932cc46c6a07b72481d9e2970 2024-11-13T18:25:15,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522375055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/a3fa055932cc46c6a07b72481d9e2970, entries=150, sequenceid=541, filesize=12.0 K 2024-11-13T18:25:15,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~403.19 KB/412864, currentSize=40.25 KB/41220 for 304012adc4150a48d7183d0af2e39078 in 757ms, sequenceid=541, compaction requested=true 2024-11-13T18:25:15,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:15,083 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:25:15,085 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:25:15,085 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:25:15,085 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. because compaction request was cancelled 2024-11-13T18:25:15,085 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:15,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:25:15,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:15,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 0 2024-11-13T18:25:15,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:15,086 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T18:25:15,086 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:15,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:15,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:15,088 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:25:15,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:15,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:15,093 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:25:15,093 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:25:15,093 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. because compaction request was cancelled 2024-11-13T18:25:15,093 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:15,095 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:15,095 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/B is initiating minor compaction (all files) 2024-11-13T18:25:15,095 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/B in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:15,095 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/f00d174780a34dc7a9195524e426f873, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/035b532fa9904907960ab5242ca879a3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e10579f251f44996b67a0e9896c0ae44] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=36.8 K 2024-11-13T18:25:15,100 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting f00d174780a34dc7a9195524e426f873, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731522312918 2024-11-13T18:25:15,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/4cfa2783123f4b269860afa7a2e302b1 is 50, key is test_row_0/A:col10/1731522314358/Put/seqid=0 2024-11-13T18:25:15,101 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 035b532fa9904907960ab5242ca879a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731522313076 2024-11-13T18:25:15,102 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting e10579f251f44996b67a0e9896c0ae44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731522313617 2024-11-13T18:25:15,136 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:15,137 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/14ff1340595f498e91b02be259226029 is 50, key is test_row_0/B:col10/1731522313648/Put/seqid=0 2024-11-13T18:25:15,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742141_1317 (size=12301) 2024-11-13T18:25:15,168 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/4cfa2783123f4b269860afa7a2e302b1 2024-11-13T18:25:15,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742142_1318 (size=13221) 2024-11-13T18:25:15,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/625ceea3cd1643d58d08044f9668804e is 50, key is test_row_0/B:col10/1731522314358/Put/seqid=0 2024-11-13T18:25:15,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742143_1319 (size=12301) 2024-11-13T18:25:15,230 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/625ceea3cd1643d58d08044f9668804e 2024-11-13T18:25:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/0fdeaf0502334884a241942b106250cf is 50, key is test_row_0/C:col10/1731522314358/Put/seqid=0 2024-11-13T18:25:15,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742144_1320 (size=12301) 2024-11-13T18:25:15,331 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/0fdeaf0502334884a241942b106250cf 2024-11-13T18:25:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/4cfa2783123f4b269860afa7a2e302b1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4cfa2783123f4b269860afa7a2e302b1 2024-11-13T18:25:15,357 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4cfa2783123f4b269860afa7a2e302b1, entries=150, sequenceid=552, filesize=12.0 K 2024-11-13T18:25:15,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/625ceea3cd1643d58d08044f9668804e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/625ceea3cd1643d58d08044f9668804e 2024-11-13T18:25:15,367 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/625ceea3cd1643d58d08044f9668804e, entries=150, sequenceid=552, filesize=12.0 K 2024-11-13T18:25:15,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/0fdeaf0502334884a241942b106250cf as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/0fdeaf0502334884a241942b106250cf 2024-11-13T18:25:15,378 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/0fdeaf0502334884a241942b106250cf, entries=150, sequenceid=552, filesize=12.0 K 2024-11-13T18:25:15,382 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 304012adc4150a48d7183d0af2e39078 in 296ms, sequenceid=552, compaction requested=true 2024-11-13T18:25:15,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:15,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:15,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-13T18:25:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-13T18:25:15,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-13T18:25:15,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9230 sec 2024-11-13T18:25:15,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9410 sec 2024-11-13T18:25:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:15,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:15,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:15,591 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/14ff1340595f498e91b02be259226029 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/14ff1340595f498e91b02be259226029 2024-11-13T18:25:15,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/fb8133368ffc491c9278b66ab410282b is 50, key is test_row_0/A:col10/1731522315571/Put/seqid=0 2024-11-13T18:25:15,602 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/B of 304012adc4150a48d7183d0af2e39078 into 14ff1340595f498e91b02be259226029(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:15,602 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:15,602 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/B, priority=13, startTime=1731522315086; duration=0sec 2024-11-13T18:25:15,602 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:15,602 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:25:15,636 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:15,637 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-13T18:25:15,640 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:25:15,640 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:15,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:15,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742145_1321 (size=12301) 2024-11-13T18:25:15,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522375645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522375649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522375650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522375655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522375657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:25:15,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522375793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522375793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522375793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522375793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:15,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:15,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522375793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:15,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:25:15,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:15,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:15,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:15,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:15,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:15,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522376006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522376007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522376007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522376015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522376015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/fb8133368ffc491c9278b66ab410282b 2024-11-13T18:25:16,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/4934d661bf8349258ae9716cc1a09051 is 50, key is test_row_0/B:col10/1731522315571/Put/seqid=0 2024-11-13T18:25:16,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742146_1322 (size=12301) 2024-11-13T18:25:16,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:16,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:16,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/4934d661bf8349258ae9716cc1a09051 2024-11-13T18:25:16,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/b5ffcfb2f9904b17adabe27197da02cd is 50, key is test_row_0/C:col10/1731522315571/Put/seqid=0 2024-11-13T18:25:16,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742147_1323 (size=12301) 2024-11-13T18:25:16,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/b5ffcfb2f9904b17adabe27197da02cd 2024-11-13T18:25:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:25:16,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/fb8133368ffc491c9278b66ab410282b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fb8133368ffc491c9278b66ab410282b 2024-11-13T18:25:16,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fb8133368ffc491c9278b66ab410282b, entries=150, sequenceid=564, filesize=12.0 K 2024-11-13T18:25:16,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/4934d661bf8349258ae9716cc1a09051 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4934d661bf8349258ae9716cc1a09051 2024-11-13T18:25:16,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4934d661bf8349258ae9716cc1a09051, entries=150, sequenceid=564, filesize=12.0 K 2024-11-13T18:25:16,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/b5ffcfb2f9904b17adabe27197da02cd as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b5ffcfb2f9904b17adabe27197da02cd 2024-11-13T18:25:16,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b5ffcfb2f9904b17adabe27197da02cd, entries=150, sequenceid=564, filesize=12.0 K 2024-11-13T18:25:16,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 304012adc4150a48d7183d0af2e39078 in 733ms, sequenceid=564, compaction requested=true 2024-11-13T18:25:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:16,310 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:16,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:16,311 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:16,311 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/A is initiating minor compaction (all files) 2024-11-13T18:25:16,311 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/A in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,312 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/0aaf711012c5414eb96750a6f690b730, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/68441d53fe7d4719839a4c564e8adca6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4cfa2783123f4b269860afa7a2e302b1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fb8133368ffc491c9278b66ab410282b] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=51.3 K 2024-11-13T18:25:16,313 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aaf711012c5414eb96750a6f690b730, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731522313076 2024-11-13T18:25:16,313 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68441d53fe7d4719839a4c564e8adca6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731522313617 2024-11-13T18:25:16,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:16,314 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cfa2783123f4b269860afa7a2e302b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731522314353 2024-11-13T18:25:16,314 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb8133368ffc491c9278b66ab410282b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731522315561 2024-11-13T18:25:16,316 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:16,317 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:16,317 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/B is initiating minor compaction (all files) 2024-11-13T18:25:16,317 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/B in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,317 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/14ff1340595f498e91b02be259226029, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/625ceea3cd1643d58d08044f9668804e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4934d661bf8349258ae9716cc1a09051] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=36.9 K 2024-11-13T18:25:16,318 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 14ff1340595f498e91b02be259226029, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731522313617 2024-11-13T18:25:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:16,319 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 625ceea3cd1643d58d08044f9668804e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731522314353 2024-11-13T18:25:16,319 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 4934d661bf8349258ae9716cc1a09051, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731522315561 2024-11-13T18:25:16,339 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#A#compaction#276 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:16,339 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/58221c34847f438a89736d7325380553 is 50, key is test_row_0/A:col10/1731522315571/Put/seqid=0 2024-11-13T18:25:16,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522315617, max timestamp=1731522316335Num uniques 859; 2024-11-13T18:25:16,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:16,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522315617, max timestamp=1731522316335Num uniques 859; 2024-11-13T18:25:16,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:16,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522315617, max timestamp=1731522316335Num uniques 859; 2024-11-13T18:25:16,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:16,349 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:16,350 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/04122b77bb4d4580a6528d306b28e3cb is 50, key is test_row_0/B:col10/1731522315571/Put/seqid=0 2024-11-13T18:25:16,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-13T18:25:16,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:16,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:16,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:16,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:16,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/bef78f2e98bf4d308411d57b44dd2217 is 50, key is test_row_0/A:col10/1731522316339/Put/seqid=0 2024-11-13T18:25:16,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742149_1325 (size=13323) 2024-11-13T18:25:16,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522316404, max timestamp=1731522316426Num uniques 859; 2024-11-13T18:25:16,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:16,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742148_1324 (size=13323) 2024-11-13T18:25:16,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522316404, max timestamp=1731522316425Num uniques 859; 2024-11-13T18:25:16,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:16,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522316404, max timestamp=1731522316425Num uniques 859; 2024-11-13T18:25:16,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:16,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:16,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742150_1326 (size=17181) 2024-11-13T18:25:16,445 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/58221c34847f438a89736d7325380553 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/58221c34847f438a89736d7325380553 2024-11-13T18:25:16,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/bef78f2e98bf4d308411d57b44dd2217 2024-11-13T18:25:16,460 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 304012adc4150a48d7183d0af2e39078/A of 304012adc4150a48d7183d0af2e39078 into 58221c34847f438a89736d7325380553(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:16,460 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:16,460 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/A, priority=12, startTime=1731522316310; duration=0sec 2024-11-13T18:25:16,460 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:16,460 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:16,460 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:16,464 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:16,465 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/C is initiating minor compaction (all files) 2024-11-13T18:25:16,465 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/C in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,465 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/305b68ca62c945f780e43d62492de916, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/a3fa055932cc46c6a07b72481d9e2970, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/0fdeaf0502334884a241942b106250cf, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b5ffcfb2f9904b17adabe27197da02cd] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=48.9 K 2024-11-13T18:25:16,465 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 305b68ca62c945f780e43d62492de916, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731522313076 2024-11-13T18:25:16,466 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3fa055932cc46c6a07b72481d9e2970, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731522313617 2024-11-13T18:25:16,466 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fdeaf0502334884a241942b106250cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731522314353 2024-11-13T18:25:16,467 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5ffcfb2f9904b17adabe27197da02cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731522315561 2024-11-13T18:25:16,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/cd65167cbd304de7b58a5dc42fd91e83 is 50, key is test_row_0/B:col10/1731522316339/Put/seqid=0 2024-11-13T18:25:16,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742151_1327 (size=12301) 2024-11-13T18:25:16,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/cd65167cbd304de7b58a5dc42fd91e83 2024-11-13T18:25:16,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731522316488, max timestamp=1731522316544Num uniques 1018; 2024-11-13T18:25:16,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T18:25:16,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731522316488, max timestamp=1731522316544Num uniques 1018; 2024-11-13T18:25:16,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T18:25:16,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731522316488, max timestamp=1731522316544Num uniques 1018; 2024-11-13T18:25:16,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T18:25:16,577 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#C#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:16,578 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/e300ab1fbc774303b319a34d25b23270 is 50, key is test_row_0/C:col10/1731522315571/Put/seqid=0 2024-11-13T18:25:16,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:16,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:16,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/86581eea5769458896bd963bd1f98d6a is 50, key is test_row_0/C:col10/1731522316339/Put/seqid=0 2024-11-13T18:25:16,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522376622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522376637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522376639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522376639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522376640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742152_1328 (size=13323) 2024-11-13T18:25:16,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742153_1329 (size=12301) 2024-11-13T18:25:16,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/86581eea5769458896bd963bd1f98d6a 2024-11-13T18:25:16,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/bef78f2e98bf4d308411d57b44dd2217 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/bef78f2e98bf4d308411d57b44dd2217 2024-11-13T18:25:16,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/bef78f2e98bf4d308411d57b44dd2217, entries=250, sequenceid=590, filesize=16.8 K 2024-11-13T18:25:16,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/cd65167cbd304de7b58a5dc42fd91e83 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/cd65167cbd304de7b58a5dc42fd91e83 2024-11-13T18:25:16,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:16,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:16,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/cd65167cbd304de7b58a5dc42fd91e83, entries=150, sequenceid=590, filesize=12.0 K 2024-11-13T18:25:16,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/86581eea5769458896bd963bd1f98d6a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/86581eea5769458896bd963bd1f98d6a 2024-11-13T18:25:16,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/86581eea5769458896bd963bd1f98d6a, entries=150, sequenceid=590, filesize=12.0 K 2024-11-13T18:25:16,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=140.89 KB/144270 for 304012adc4150a48d7183d0af2e39078 in 425ms, sequenceid=590, compaction requested=false 2024-11-13T18:25:16,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:25:16,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-13T18:25:16,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:16,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:25:16,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:16,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:25:16,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:16,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:25:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:16,826 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/04122b77bb4d4580a6528d306b28e3cb as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04122b77bb4d4580a6528d306b28e3cb 2024-11-13T18:25:16,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522376807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,840 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/B of 304012adc4150a48d7183d0af2e39078 into 04122b77bb4d4580a6528d306b28e3cb(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:16,840 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:16,840 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/B, priority=13, startTime=1731522316316; duration=0sec 2024-11-13T18:25:16,840 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:16,840 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:16,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522376816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/fef602f2256a48f8aeca43ced28e30af is 50, key is test_row_0/A:col10/1731522316637/Put/seqid=0 2024-11-13T18:25:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522376818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522376824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522376834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742154_1330 (size=14741) 2024-11-13T18:25:16,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/fef602f2256a48f8aeca43ced28e30af 2024-11-13T18:25:16,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:16,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:16,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a29b1acc940f4647be476f22ec190917 is 50, key is test_row_0/B:col10/1731522316637/Put/seqid=0 2024-11-13T18:25:16,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522376939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522376947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522376955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522376966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522376966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:16,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742155_1331 (size=12301) 2024-11-13T18:25:16,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a29b1acc940f4647be476f22ec190917 2024-11-13T18:25:17,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/9235b07b399644a9846a313dd4907f37 is 50, key is test_row_0/C:col10/1731522316637/Put/seqid=0 2024-11-13T18:25:17,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742156_1332 (size=12301) 2024-11-13T18:25:17,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/9235b07b399644a9846a313dd4907f37 2024-11-13T18:25:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/fef602f2256a48f8aeca43ced28e30af as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fef602f2256a48f8aeca43ced28e30af 2024-11-13T18:25:17,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:17,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:17,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fef602f2256a48f8aeca43ced28e30af, entries=200, sequenceid=646, filesize=14.4 K 2024-11-13T18:25:17,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a29b1acc940f4647be476f22ec190917 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a29b1acc940f4647be476f22ec190917 2024-11-13T18:25:17,100 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/e300ab1fbc774303b319a34d25b23270 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/e300ab1fbc774303b319a34d25b23270 2024-11-13T18:25:17,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a29b1acc940f4647be476f22ec190917, entries=150, sequenceid=646, filesize=12.0 K 2024-11-13T18:25:17,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/9235b07b399644a9846a313dd4907f37 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9235b07b399644a9846a313dd4907f37 2024-11-13T18:25:17,126 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 304012adc4150a48d7183d0af2e39078/C of 304012adc4150a48d7183d0af2e39078 into e300ab1fbc774303b319a34d25b23270(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:17,126 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:17,126 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/C, priority=12, startTime=1731522316318; duration=0sec 2024-11-13T18:25:17,126 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:17,126 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:17,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9235b07b399644a9846a313dd4907f37, entries=150, sequenceid=646, filesize=12.0 K 2024-11-13T18:25:17,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 304012adc4150a48d7183d0af2e39078 in 345ms, sequenceid=646, compaction requested=true 2024-11-13T18:25:17,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:17,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:17,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:17,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:17,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:17,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 304012adc4150a48d7183d0af2e39078:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:17,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:17,149 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:17,149 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:17,155 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:17,155 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/A is initiating minor compaction (all files) 2024-11-13T18:25:17,155 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/A in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,155 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/58221c34847f438a89736d7325380553, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/bef78f2e98bf4d308411d57b44dd2217, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fef602f2256a48f8aeca43ced28e30af] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=44.2 K 2024-11-13T18:25:17,156 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:17,156 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/C is initiating minor compaction (all files) 2024-11-13T18:25:17,156 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58221c34847f438a89736d7325380553, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731522315561 2024-11-13T18:25:17,157 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/C in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,157 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/e300ab1fbc774303b319a34d25b23270, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/86581eea5769458896bd963bd1f98d6a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9235b07b399644a9846a313dd4907f37] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=37.0 K 2024-11-13T18:25:17,160 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e300ab1fbc774303b319a34d25b23270, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731522315561 2024-11-13T18:25:17,160 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting bef78f2e98bf4d308411d57b44dd2217, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731522316335 2024-11-13T18:25:17,162 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 86581eea5769458896bd963bd1f98d6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731522316339 2024-11-13T18:25:17,162 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting fef602f2256a48f8aeca43ced28e30af, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1731522316608 2024-11-13T18:25:17,162 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9235b07b399644a9846a313dd4907f37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1731522316608 2024-11-13T18:25:17,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:17,183 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:17,184 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/02eb1821c15145598e9a15d06a433910 is 50, key is test_row_0/C:col10/1731522316637/Put/seqid=0 2024-11-13T18:25:17,198 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#A#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:17,199 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/feb9e330fe214d308be6a30c7df9f5b3 is 50, key is test_row_0/A:col10/1731522316637/Put/seqid=0 2024-11-13T18:25:17,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:17,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:17,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742157_1333 (size=13425) 2024-11-13T18:25:17,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/e0a1f363524a4b3bb162e6853211620f is 50, key is test_row_0/A:col10/1731522317163/Put/seqid=0 2024-11-13T18:25:17,292 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/02eb1821c15145598e9a15d06a433910 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/02eb1821c15145598e9a15d06a433910 2024-11-13T18:25:17,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742158_1334 (size=13425) 2024-11-13T18:25:17,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742159_1335 (size=14741) 2024-11-13T18:25:17,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522377270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522377271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/e0a1f363524a4b3bb162e6853211620f 2024-11-13T18:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,349 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/C of 304012adc4150a48d7183d0af2e39078 into 02eb1821c15145598e9a15d06a433910(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:17,349 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:17,349 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/C, priority=13, startTime=1731522317149; duration=0sec 2024-11-13T18:25:17,349 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:17,349 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:C 2024-11-13T18:25:17,350 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:17,353 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:17,354 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 304012adc4150a48d7183d0af2e39078/B is initiating minor compaction (all files) 2024-11-13T18:25:17,354 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 304012adc4150a48d7183d0af2e39078/B in TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,354 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04122b77bb4d4580a6528d306b28e3cb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/cd65167cbd304de7b58a5dc42fd91e83, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a29b1acc940f4647be476f22ec190917] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp, totalSize=37.0 K 2024-11-13T18:25:17,354 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 04122b77bb4d4580a6528d306b28e3cb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731522315561 2024-11-13T18:25:17,355 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting cd65167cbd304de7b58a5dc42fd91e83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731522316339 2024-11-13T18:25:17,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522377331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,362 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting a29b1acc940f4647be476f22ec190917, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1731522316608 2024-11-13T18:25:17,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522377351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/9307d237c6fe47999a2ef7154db02feb is 50, key is test_row_0/B:col10/1731522317163/Put/seqid=0 2024-11-13T18:25:17,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522377353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,390 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 304012adc4150a48d7183d0af2e39078#B#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:17,391 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/741c318f462a4ef283f6f299d0755340 is 50, key is test_row_0/B:col10/1731522316637/Put/seqid=0 2024-11-13T18:25:17,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:17,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:17,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,431 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-13T18:25:17,431 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:17,431 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,431 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,431 INFO [Thread-1116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1116. 2024-11-13T18:25:17,432 DEBUG [Thread-1116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:17,432 DEBUG [Thread-1116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,432 DEBUG [Thread-1116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:17,432 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-11-13T18:25:17,432 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:17,432 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,432 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:17,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:17,439 INFO [Thread-1114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1114. 2024-11-13T18:25:17,439 DEBUG [Thread-1114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:17,439 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,439 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742161_1337 (size=13425) 2024-11-13T18:25:17,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:17,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742160_1336 (size=12301) 2024-11-13T18:25:17,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522377449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522377449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/9307d237c6fe47999a2ef7154db02feb 2024-11-13T18:25:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,459 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-13T18:25:17,459 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:17,459 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,460 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,460 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522377466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,472 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/741c318f462a4ef283f6f299d0755340 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/741c318f462a4ef283f6f299d0755340 2024-11-13T18:25:17,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522377475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522377486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,491 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/B of 304012adc4150a48d7183d0af2e39078 into 741c318f462a4ef283f6f299d0755340(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:17,491 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:17,491 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/B, priority=13, startTime=1731522317148; duration=0sec 2024-11-13T18:25:17,491 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:17,491 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:B 2024-11-13T18:25:17,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/652943b50941417ab7b09686a351c444 is 50, key is test_row_0/C:col10/1731522317163/Put/seqid=0 2024-11-13T18:25:17,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:17,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:17,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742162_1338 (size=12301) 2024-11-13T18:25:17,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/652943b50941417ab7b09686a351c444 2024-11-13T18:25:17,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/e0a1f363524a4b3bb162e6853211620f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/e0a1f363524a4b3bb162e6853211620f 2024-11-13T18:25:17,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/e0a1f363524a4b3bb162e6853211620f, entries=200, sequenceid=659, filesize=14.4 K 2024-11-13T18:25:17,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43396 deadline: 1731522377655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/9307d237c6fe47999a2ef7154db02feb as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/9307d237c6fe47999a2ef7154db02feb 2024-11-13T18:25:17,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43424 deadline: 1731522377666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43478 deadline: 1731522377676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43432 deadline: 1731522377687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/9307d237c6fe47999a2ef7154db02feb, entries=150, sequenceid=659, filesize=12.0 K 2024-11-13T18:25:17,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43440 deadline: 1731522377697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/652943b50941417ab7b09686a351c444 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/652943b50941417ab7b09686a351c444 2024-11-13T18:25:17,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/652943b50941417ab7b09686a351c444, entries=150, sequenceid=659, filesize=12.0 K 2024-11-13T18:25:17,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:17,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 304012adc4150a48d7183d0af2e39078 in 552ms, sequenceid=659, compaction requested=false 2024-11-13T18:25:17,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:17,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:17,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:17,766 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/feb9e330fe214d308be6a30c7df9f5b3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/feb9e330fe214d308be6a30c7df9f5b3 2024-11-13T18:25:17,787 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 304012adc4150a48d7183d0af2e39078/A of 304012adc4150a48d7183d0af2e39078 into feb9e330fe214d308be6a30c7df9f5b3(size=13.1 K), total size for store is 27.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:17,787 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:17,787 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., storeName=304012adc4150a48d7183d0af2e39078/A, priority=13, startTime=1731522317148; duration=0sec 2024-11-13T18:25:17,787 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:17,787 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 304012adc4150a48d7183d0af2e39078:A 2024-11-13T18:25:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:25:17,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:25:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:17,884 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-13T18:25:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:17,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/fd080d0ee6d74ed1be793f7bf5a03258 is 50, key is test_row_0/A:col10/1731522317301/Put/seqid=0 2024-11-13T18:25:17,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742163_1339 (size=12301) 2024-11-13T18:25:17,959 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=685 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/fd080d0ee6d74ed1be793f7bf5a03258 2024-11-13T18:25:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:17,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. as already flushing 2024-11-13T18:25:17,971 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-11-13T18:25:17,971 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:17,971 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,971 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:17,978 INFO [Thread-1105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1105. 2024-11-13T18:25:17,978 DEBUG [Thread-1105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:17,978 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,978 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:17,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/c50968c07c5045a595e051d08022fdea is 50, key is test_row_0/B:col10/1731522317301/Put/seqid=0 2024-11-13T18:25:17,995 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-11-13T18:25:17,995 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:17,995 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,995 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:17,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:18,007 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-11-13T18:25:18,007 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:18,007 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:18,007 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:18,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:18,010 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-11-13T18:25:18,010 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:18,010 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:18,010 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:18,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:18,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742164_1340 (size=12301) 2024-11-13T18:25:18,066 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=685 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/c50968c07c5045a595e051d08022fdea 2024-11-13T18:25:18,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/d800b0b0481d4d97bc9fcae9fa50106c is 50, key is test_row_0/C:col10/1731522317301/Put/seqid=0 2024-11-13T18:25:18,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742165_1341 (size=12301) 2024-11-13T18:25:18,183 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=685 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/d800b0b0481d4d97bc9fcae9fa50106c 2024-11-13T18:25:18,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/fd080d0ee6d74ed1be793f7bf5a03258 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fd080d0ee6d74ed1be793f7bf5a03258 2024-11-13T18:25:18,243 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fd080d0ee6d74ed1be793f7bf5a03258, entries=150, sequenceid=685, filesize=12.0 K 2024-11-13T18:25:18,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/c50968c07c5045a595e051d08022fdea as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/c50968c07c5045a595e051d08022fdea 2024-11-13T18:25:18,280 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/c50968c07c5045a595e051d08022fdea, entries=150, sequenceid=685, filesize=12.0 K 2024-11-13T18:25:18,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/d800b0b0481d4d97bc9fcae9fa50106c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/d800b0b0481d4d97bc9fcae9fa50106c 2024-11-13T18:25:18,304 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/d800b0b0481d4d97bc9fcae9fa50106c, entries=150, sequenceid=685, filesize=12.0 K 2024-11-13T18:25:18,306 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=33.54 KB/34350 for 304012adc4150a48d7183d0af2e39078 in 422ms, sequenceid=685, compaction requested=true 2024-11-13T18:25:18,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 304012adc4150a48d7183d0af2e39078: 2024-11-13T18:25:18,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:18,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-13T18:25:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-13T18:25:18,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-13T18:25:18,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6680 sec 2024-11-13T18:25:18,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.6770 sec 2024-11-13T18:25:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:25:19,830 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 143 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1354 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1228 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1222 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1317 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1244 2024-11-13T18:25:19,831 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:25:19,831 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:25:19,831 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:25:19,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:19,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:19,832 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:25:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:19,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:19,837 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522319836"}]},"ts":"1731522319836"} 2024-11-13T18:25:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:25:19,851 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:25:19,851 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:25:19,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:25:19,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=304012adc4150a48d7183d0af2e39078, UNASSIGN}] 2024-11-13T18:25:19,860 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=304012adc4150a48d7183d0af2e39078, UNASSIGN 2024-11-13T18:25:19,861 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=304012adc4150a48d7183d0af2e39078, regionState=CLOSING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:25:19,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=304012adc4150a48d7183d0af2e39078, UNASSIGN because future has completed 2024-11-13T18:25:19,875 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:25:19,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:25:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:25:20,034 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:20,034 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:25:20,035 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing 304012adc4150a48d7183d0af2e39078, disabling compactions & flushes 2024-11-13T18:25:20,035 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:20,035 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:20,035 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. after waiting 0 ms 2024-11-13T18:25:20,035 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:20,035 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing 304012adc4150a48d7183d0af2e39078 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T18:25:20,035 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=A 2024-11-13T18:25:20,035 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:20,035 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=B 2024-11-13T18:25:20,035 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:20,036 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 304012adc4150a48d7183d0af2e39078, store=C 2024-11-13T18:25:20,036 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:20,060 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/99c2f462ac5d4c7ca9911a2f99310bc5 is 50, key is test_row_0/A:col10/1731522318008/Put/seqid=0 2024-11-13T18:25:20,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742166_1342 (size=12301) 2024-11-13T18:25:20,103 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/99c2f462ac5d4c7ca9911a2f99310bc5 2024-11-13T18:25:20,122 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a277ee651f914096ab3f40a80fe8f216 is 50, key is test_row_0/B:col10/1731522318008/Put/seqid=0 2024-11-13T18:25:20,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742167_1343 (size=12301) 2024-11-13T18:25:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:25:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:25:20,549 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a277ee651f914096ab3f40a80fe8f216 2024-11-13T18:25:20,580 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/fa55cb32fa594fe4b82f6e75cab5c031 is 50, key is test_row_0/C:col10/1731522318008/Put/seqid=0 2024-11-13T18:25:20,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742168_1344 (size=12301) 2024-11-13T18:25:20,601 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/fa55cb32fa594fe4b82f6e75cab5c031 2024-11-13T18:25:20,607 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/A/99c2f462ac5d4c7ca9911a2f99310bc5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/99c2f462ac5d4c7ca9911a2f99310bc5 2024-11-13T18:25:20,612 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/99c2f462ac5d4c7ca9911a2f99310bc5, entries=150, sequenceid=693, filesize=12.0 K 2024-11-13T18:25:20,616 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/B/a277ee651f914096ab3f40a80fe8f216 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a277ee651f914096ab3f40a80fe8f216 2024-11-13T18:25:20,627 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a277ee651f914096ab3f40a80fe8f216, entries=150, sequenceid=693, filesize=12.0 K 2024-11-13T18:25:20,633 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/.tmp/C/fa55cb32fa594fe4b82f6e75cab5c031 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/fa55cb32fa594fe4b82f6e75cab5c031 2024-11-13T18:25:20,657 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/fa55cb32fa594fe4b82f6e75cab5c031, entries=150, sequenceid=693, filesize=12.0 K 2024-11-13T18:25:20,662 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 304012adc4150a48d7183d0af2e39078 in 626ms, sequenceid=693, compaction requested=true 2024-11-13T18:25:20,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d2753ef05c884baa8cc7ab04ce36b74d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a701f9a1c914634a16b96d5170b48a2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/947f24a9061c4462bda7a11d1c4a046f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/ea9eeca33c7945c5a1b4e649e5567ee3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/a74f682396764abba595a96b87de0b33, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/dd676c2fd41f4af1aa0cc481a02fdc24, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/187460c88b9141e7a6d01de97eadcaf9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/8efaa9d9cb4d4fcd8588315441639d0b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5d67cef49e9a41c6bb0546760f172c98, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/e28f6c5d5d8649e5b67224c441e20ad9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cfb7479b134f41b8abd8c6b7aec76b3f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4c6ccef1315e48d29ec8697dfa5e810f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9cc60ca98d0b48158666a6ae624d38dd, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5da596225f9f4a7b9c7e512402a16019, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/219138a2a88942788d2a46708df3e92f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/242825db7535495c977db04d586bb001, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/7f97ad35737f48ddaef94535d3a5ccb9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/259a6ec85d7d462ab90bd0a5f3f61a26, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/15fde0ed03a54db8bf64af23bc2bee8f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5894221438924d39b8d3eb38e55c8f89, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d7f097ad8a924d75946a2bf144d4d1cd, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/94b6010e4a2a43778f9b52dbd50c675c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5952178bd3464a589542f0c4afb63057, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a4ecf65d16748ce9fce2b06cafee3fd, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cb2f8a09225845dcb45cef09ac8675d1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5aae966e70b0447ab74bd95040f37676, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/0aaf711012c5414eb96750a6f690b730, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/68441d53fe7d4719839a4c564e8adca6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4cfa2783123f4b269860afa7a2e302b1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/58221c34847f438a89736d7325380553, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fb8133368ffc491c9278b66ab410282b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/bef78f2e98bf4d308411d57b44dd2217, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fef602f2256a48f8aeca43ced28e30af] to archive 2024-11-13T18:25:20,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:20,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d2753ef05c884baa8cc7ab04ce36b74d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d2753ef05c884baa8cc7ab04ce36b74d 2024-11-13T18:25:20,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a701f9a1c914634a16b96d5170b48a2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a701f9a1c914634a16b96d5170b48a2 2024-11-13T18:25:20,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/947f24a9061c4462bda7a11d1c4a046f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/947f24a9061c4462bda7a11d1c4a046f 2024-11-13T18:25:20,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/ea9eeca33c7945c5a1b4e649e5567ee3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/ea9eeca33c7945c5a1b4e649e5567ee3 2024-11-13T18:25:20,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/a74f682396764abba595a96b87de0b33 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/a74f682396764abba595a96b87de0b33 2024-11-13T18:25:20,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/dd676c2fd41f4af1aa0cc481a02fdc24 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/dd676c2fd41f4af1aa0cc481a02fdc24 2024-11-13T18:25:20,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/187460c88b9141e7a6d01de97eadcaf9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/187460c88b9141e7a6d01de97eadcaf9 2024-11-13T18:25:20,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/8efaa9d9cb4d4fcd8588315441639d0b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/8efaa9d9cb4d4fcd8588315441639d0b 2024-11-13T18:25:20,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5d67cef49e9a41c6bb0546760f172c98 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5d67cef49e9a41c6bb0546760f172c98 2024-11-13T18:25:20,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/e28f6c5d5d8649e5b67224c441e20ad9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/e28f6c5d5d8649e5b67224c441e20ad9 2024-11-13T18:25:20,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cfb7479b134f41b8abd8c6b7aec76b3f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cfb7479b134f41b8abd8c6b7aec76b3f 2024-11-13T18:25:20,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4c6ccef1315e48d29ec8697dfa5e810f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4c6ccef1315e48d29ec8697dfa5e810f 2024-11-13T18:25:20,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9cc60ca98d0b48158666a6ae624d38dd to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9cc60ca98d0b48158666a6ae624d38dd 2024-11-13T18:25:20,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5da596225f9f4a7b9c7e512402a16019 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5da596225f9f4a7b9c7e512402a16019 2024-11-13T18:25:20,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/219138a2a88942788d2a46708df3e92f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/219138a2a88942788d2a46708df3e92f 2024-11-13T18:25:20,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/242825db7535495c977db04d586bb001 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/242825db7535495c977db04d586bb001 2024-11-13T18:25:20,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/7f97ad35737f48ddaef94535d3a5ccb9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/7f97ad35737f48ddaef94535d3a5ccb9 2024-11-13T18:25:20,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/259a6ec85d7d462ab90bd0a5f3f61a26 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/259a6ec85d7d462ab90bd0a5f3f61a26 2024-11-13T18:25:20,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/15fde0ed03a54db8bf64af23bc2bee8f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/15fde0ed03a54db8bf64af23bc2bee8f 2024-11-13T18:25:20,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5894221438924d39b8d3eb38e55c8f89 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5894221438924d39b8d3eb38e55c8f89 2024-11-13T18:25:20,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d7f097ad8a924d75946a2bf144d4d1cd to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/d7f097ad8a924d75946a2bf144d4d1cd 2024-11-13T18:25:20,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/94b6010e4a2a43778f9b52dbd50c675c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/94b6010e4a2a43778f9b52dbd50c675c 2024-11-13T18:25:20,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5952178bd3464a589542f0c4afb63057 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5952178bd3464a589542f0c4afb63057 2024-11-13T18:25:20,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a4ecf65d16748ce9fce2b06cafee3fd to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/9a4ecf65d16748ce9fce2b06cafee3fd 2024-11-13T18:25:20,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cb2f8a09225845dcb45cef09ac8675d1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/cb2f8a09225845dcb45cef09ac8675d1 2024-11-13T18:25:20,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5aae966e70b0447ab74bd95040f37676 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/5aae966e70b0447ab74bd95040f37676 2024-11-13T18:25:20,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/0aaf711012c5414eb96750a6f690b730 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/0aaf711012c5414eb96750a6f690b730 2024-11-13T18:25:20,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/68441d53fe7d4719839a4c564e8adca6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/68441d53fe7d4719839a4c564e8adca6 2024-11-13T18:25:20,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4cfa2783123f4b269860afa7a2e302b1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/4cfa2783123f4b269860afa7a2e302b1 2024-11-13T18:25:20,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/58221c34847f438a89736d7325380553 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/58221c34847f438a89736d7325380553 2024-11-13T18:25:20,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fb8133368ffc491c9278b66ab410282b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fb8133368ffc491c9278b66ab410282b 2024-11-13T18:25:20,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/bef78f2e98bf4d308411d57b44dd2217 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/bef78f2e98bf4d308411d57b44dd2217 2024-11-13T18:25:20,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fef602f2256a48f8aeca43ced28e30af to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fef602f2256a48f8aeca43ced28e30af 2024-11-13T18:25:20,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/60cfd8578fa4458095d28d184f022b44, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/70828f7be70a4a92ba5b580b0f0bb12f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d9a48ef16d254de09c527082c0326a20, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/fb4b69f4945643f98f046cebe943d176, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ef074afe7de44fe68e9743ac99a88cc9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a488f206979d474bbd6c51de1aaffcc3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/47f78cc7864e4641910cc60c4a958907, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/414edb91886843bd8d0a75b36dcc4446, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/f39d83cdb1414f4780f57533296aa890, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/efaa42408e9f422886b61b11f8233b90, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4c0bb97af17c48538cf1332d375a0a87, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/195f784553f54c9bb2658645124ffcfd, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e89bf4ed69b24734b98097f506e111e1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d5dbb468950f4f8a953117b81f539074, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/84129a1cf5744a9cbcf4adbcf791d179, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/bb02e4ca01d845b799b06e765d1795a9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/00b50a76f23f4ea087a290f3a2527cb4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/08dbc8e994304fd3a0b2338263125586, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a6bb87a5fe48400daaf54b50f80c9d1a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/0e37f5727b1a48a9a854cd50ced15f7c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/9acb344936ac497082334ab3a146f862, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/740a314ef6c944a4b23787ab336969fe, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04e4e05a7dac42d6aaa16473f17af839, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/f00d174780a34dc7a9195524e426f873, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ba544a0ae11a4f0ebda41a561ea29575, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/035b532fa9904907960ab5242ca879a3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/14ff1340595f498e91b02be259226029, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e10579f251f44996b67a0e9896c0ae44, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/625ceea3cd1643d58d08044f9668804e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04122b77bb4d4580a6528d306b28e3cb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4934d661bf8349258ae9716cc1a09051, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/cd65167cbd304de7b58a5dc42fd91e83, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a29b1acc940f4647be476f22ec190917] to archive 2024-11-13T18:25:20,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:20,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/60cfd8578fa4458095d28d184f022b44 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/60cfd8578fa4458095d28d184f022b44 2024-11-13T18:25:20,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/70828f7be70a4a92ba5b580b0f0bb12f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/70828f7be70a4a92ba5b580b0f0bb12f 2024-11-13T18:25:20,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d9a48ef16d254de09c527082c0326a20 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d9a48ef16d254de09c527082c0326a20 2024-11-13T18:25:20,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/fb4b69f4945643f98f046cebe943d176 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/fb4b69f4945643f98f046cebe943d176 2024-11-13T18:25:20,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ef074afe7de44fe68e9743ac99a88cc9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ef074afe7de44fe68e9743ac99a88cc9 2024-11-13T18:25:20,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a488f206979d474bbd6c51de1aaffcc3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a488f206979d474bbd6c51de1aaffcc3 2024-11-13T18:25:20,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/47f78cc7864e4641910cc60c4a958907 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/47f78cc7864e4641910cc60c4a958907 2024-11-13T18:25:20,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/414edb91886843bd8d0a75b36dcc4446 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/414edb91886843bd8d0a75b36dcc4446 2024-11-13T18:25:20,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/f39d83cdb1414f4780f57533296aa890 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/f39d83cdb1414f4780f57533296aa890 2024-11-13T18:25:20,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/efaa42408e9f422886b61b11f8233b90 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/efaa42408e9f422886b61b11f8233b90 2024-11-13T18:25:20,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4c0bb97af17c48538cf1332d375a0a87 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4c0bb97af17c48538cf1332d375a0a87 2024-11-13T18:25:20,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/195f784553f54c9bb2658645124ffcfd to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/195f784553f54c9bb2658645124ffcfd 2024-11-13T18:25:20,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e89bf4ed69b24734b98097f506e111e1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e89bf4ed69b24734b98097f506e111e1 2024-11-13T18:25:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:25:21,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d5dbb468950f4f8a953117b81f539074 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/d5dbb468950f4f8a953117b81f539074 2024-11-13T18:25:21,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/84129a1cf5744a9cbcf4adbcf791d179 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/84129a1cf5744a9cbcf4adbcf791d179 2024-11-13T18:25:21,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/bb02e4ca01d845b799b06e765d1795a9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/bb02e4ca01d845b799b06e765d1795a9 2024-11-13T18:25:21,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/00b50a76f23f4ea087a290f3a2527cb4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/00b50a76f23f4ea087a290f3a2527cb4 2024-11-13T18:25:21,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/08dbc8e994304fd3a0b2338263125586 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/08dbc8e994304fd3a0b2338263125586 2024-11-13T18:25:21,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a6bb87a5fe48400daaf54b50f80c9d1a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a6bb87a5fe48400daaf54b50f80c9d1a 2024-11-13T18:25:21,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/0e37f5727b1a48a9a854cd50ced15f7c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/0e37f5727b1a48a9a854cd50ced15f7c 2024-11-13T18:25:21,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/9acb344936ac497082334ab3a146f862 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/9acb344936ac497082334ab3a146f862 2024-11-13T18:25:21,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/740a314ef6c944a4b23787ab336969fe to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/740a314ef6c944a4b23787ab336969fe 2024-11-13T18:25:21,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04e4e05a7dac42d6aaa16473f17af839 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04e4e05a7dac42d6aaa16473f17af839 2024-11-13T18:25:21,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/f00d174780a34dc7a9195524e426f873 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/f00d174780a34dc7a9195524e426f873 2024-11-13T18:25:21,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ba544a0ae11a4f0ebda41a561ea29575 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/ba544a0ae11a4f0ebda41a561ea29575 2024-11-13T18:25:21,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/035b532fa9904907960ab5242ca879a3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/035b532fa9904907960ab5242ca879a3 2024-11-13T18:25:21,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/14ff1340595f498e91b02be259226029 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/14ff1340595f498e91b02be259226029 2024-11-13T18:25:21,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e10579f251f44996b67a0e9896c0ae44 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/e10579f251f44996b67a0e9896c0ae44 2024-11-13T18:25:21,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/625ceea3cd1643d58d08044f9668804e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/625ceea3cd1643d58d08044f9668804e 2024-11-13T18:25:21,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04122b77bb4d4580a6528d306b28e3cb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/04122b77bb4d4580a6528d306b28e3cb 2024-11-13T18:25:21,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4934d661bf8349258ae9716cc1a09051 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/4934d661bf8349258ae9716cc1a09051 2024-11-13T18:25:21,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/cd65167cbd304de7b58a5dc42fd91e83 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/cd65167cbd304de7b58a5dc42fd91e83 2024-11-13T18:25:21,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a29b1acc940f4647be476f22ec190917 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a29b1acc940f4647be476f22ec190917 2024-11-13T18:25:21,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/934e20589a0b4374b4b30834f0f13c36, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8fb8d5e84694434dad284ef3f5311c6a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/dc2217d663284a529334c4bd0bb5a05c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d1315091ccc40df808c3843d543097c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/3de65910a5f5440e8edd83f053ec9cc8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d3cd5f9796b4dfb9c315a370be98ecb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/23eb4e7bb7384f0e991934f8dc9d4b76, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/279066480573440c9c24af55f3708d1e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/76f55f36db9d4510a53df2472730c1f0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b8eb246bcc94a6c9e09b166ee325e4a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/6bb451e0547b4ef3b606a6e098567f23, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/91f6ec9ea61b43739de068cc8600b997, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b0dd84ae27e346458db0bd507608cd45, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ba4e1bdb302a4d78963ae51a71559eb7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ef637fcb124c429b96b02899bf94a53a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2d14626423014214a6cf76d9e631754e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/c9ef1db1030a41a2bf9b7e18341c6769, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2009e175c48c4cc6a3ceef44a64d2483, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b07d03625ec4681a3d81bd03ab662a0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/cc5894e83f0647049ffc0bc6c8490632, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/58166833ec1b4b2ba99efca80eb6663b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/714514cb34c74028aa2a711c7be53203, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9426f60a1a24415caf2e52b2333520a0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/061991408bed4f27bf4032feb2ed599e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8da99c76217049a2828309195667a538, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/305b68ca62c945f780e43d62492de916, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/014bf73db5bf452f929b1fac5fdcf66d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/a3fa055932cc46c6a07b72481d9e2970, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/0fdeaf0502334884a241942b106250cf, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/e300ab1fbc774303b319a34d25b23270, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b5ffcfb2f9904b17adabe27197da02cd, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/86581eea5769458896bd963bd1f98d6a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9235b07b399644a9846a313dd4907f37] to archive 2024-11-13T18:25:21,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:21,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/934e20589a0b4374b4b30834f0f13c36 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/934e20589a0b4374b4b30834f0f13c36 2024-11-13T18:25:21,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8fb8d5e84694434dad284ef3f5311c6a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8fb8d5e84694434dad284ef3f5311c6a 2024-11-13T18:25:21,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/dc2217d663284a529334c4bd0bb5a05c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/dc2217d663284a529334c4bd0bb5a05c 2024-11-13T18:25:21,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d1315091ccc40df808c3843d543097c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d1315091ccc40df808c3843d543097c 2024-11-13T18:25:21,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/3de65910a5f5440e8edd83f053ec9cc8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/3de65910a5f5440e8edd83f053ec9cc8 2024-11-13T18:25:21,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d3cd5f9796b4dfb9c315a370be98ecb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/1d3cd5f9796b4dfb9c315a370be98ecb 2024-11-13T18:25:21,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/23eb4e7bb7384f0e991934f8dc9d4b76 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/23eb4e7bb7384f0e991934f8dc9d4b76 2024-11-13T18:25:21,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/279066480573440c9c24af55f3708d1e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/279066480573440c9c24af55f3708d1e 2024-11-13T18:25:21,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/76f55f36db9d4510a53df2472730c1f0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/76f55f36db9d4510a53df2472730c1f0 2024-11-13T18:25:21,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b8eb246bcc94a6c9e09b166ee325e4a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b8eb246bcc94a6c9e09b166ee325e4a 2024-11-13T18:25:21,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/6bb451e0547b4ef3b606a6e098567f23 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/6bb451e0547b4ef3b606a6e098567f23 2024-11-13T18:25:21,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/91f6ec9ea61b43739de068cc8600b997 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/91f6ec9ea61b43739de068cc8600b997 2024-11-13T18:25:21,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b0dd84ae27e346458db0bd507608cd45 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b0dd84ae27e346458db0bd507608cd45 2024-11-13T18:25:21,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ba4e1bdb302a4d78963ae51a71559eb7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ba4e1bdb302a4d78963ae51a71559eb7 2024-11-13T18:25:21,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ef637fcb124c429b96b02899bf94a53a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/ef637fcb124c429b96b02899bf94a53a 2024-11-13T18:25:21,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2d14626423014214a6cf76d9e631754e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2d14626423014214a6cf76d9e631754e 2024-11-13T18:25:21,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/c9ef1db1030a41a2bf9b7e18341c6769 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/c9ef1db1030a41a2bf9b7e18341c6769 2024-11-13T18:25:21,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2009e175c48c4cc6a3ceef44a64d2483 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/2009e175c48c4cc6a3ceef44a64d2483 2024-11-13T18:25:21,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b07d03625ec4681a3d81bd03ab662a0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/4b07d03625ec4681a3d81bd03ab662a0 2024-11-13T18:25:21,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/cc5894e83f0647049ffc0bc6c8490632 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/cc5894e83f0647049ffc0bc6c8490632 2024-11-13T18:25:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/58166833ec1b4b2ba99efca80eb6663b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/58166833ec1b4b2ba99efca80eb6663b 2024-11-13T18:25:21,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/714514cb34c74028aa2a711c7be53203 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/714514cb34c74028aa2a711c7be53203 2024-11-13T18:25:21,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9426f60a1a24415caf2e52b2333520a0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9426f60a1a24415caf2e52b2333520a0 2024-11-13T18:25:21,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/061991408bed4f27bf4032feb2ed599e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/061991408bed4f27bf4032feb2ed599e 2024-11-13T18:25:21,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8da99c76217049a2828309195667a538 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/8da99c76217049a2828309195667a538 2024-11-13T18:25:21,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/305b68ca62c945f780e43d62492de916 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/305b68ca62c945f780e43d62492de916 2024-11-13T18:25:21,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/014bf73db5bf452f929b1fac5fdcf66d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/014bf73db5bf452f929b1fac5fdcf66d 2024-11-13T18:25:21,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/a3fa055932cc46c6a07b72481d9e2970 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/a3fa055932cc46c6a07b72481d9e2970 2024-11-13T18:25:21,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/0fdeaf0502334884a241942b106250cf to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/0fdeaf0502334884a241942b106250cf 2024-11-13T18:25:21,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/e300ab1fbc774303b319a34d25b23270 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/e300ab1fbc774303b319a34d25b23270 2024-11-13T18:25:21,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b5ffcfb2f9904b17adabe27197da02cd to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/b5ffcfb2f9904b17adabe27197da02cd 2024-11-13T18:25:21,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/86581eea5769458896bd963bd1f98d6a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/86581eea5769458896bd963bd1f98d6a 2024-11-13T18:25:21,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9235b07b399644a9846a313dd4907f37 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/9235b07b399644a9846a313dd4907f37 2024-11-13T18:25:21,432 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/recovered.edits/696.seqid, newMaxSeqId=696, maxSeqId=1 2024-11-13T18:25:21,433 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. 2024-11-13T18:25:21,433 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for 304012adc4150a48d7183d0af2e39078: Waiting for close lock at 1731522320035Running coprocessor pre-close hooks at 1731522320035Disabling compacts and flushes for region at 1731522320035Disabling writes for close at 1731522320035Obtaining lock to block concurrent updates at 1731522320035Preparing flush snapshotting stores in 304012adc4150a48d7183d0af2e39078 at 1731522320035Finished memstore snapshotting TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731522320036 (+1 ms)Flushing stores of TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078. at 1731522320043 (+7 ms)Flushing 304012adc4150a48d7183d0af2e39078/A: creating writer at 1731522320043Flushing 304012adc4150a48d7183d0af2e39078/A: appending metadata at 1731522320060 (+17 ms)Flushing 304012adc4150a48d7183d0af2e39078/A: closing flushed file at 1731522320060Flushing 304012adc4150a48d7183d0af2e39078/B: creating writer at 1731522320107 (+47 ms)Flushing 304012adc4150a48d7183d0af2e39078/B: appending metadata at 1731522320121 (+14 ms)Flushing 304012adc4150a48d7183d0af2e39078/B: closing flushed file at 1731522320121Flushing 304012adc4150a48d7183d0af2e39078/C: creating writer at 1731522320564 (+443 ms)Flushing 304012adc4150a48d7183d0af2e39078/C: appending metadata at 1731522320579 (+15 ms)Flushing 304012adc4150a48d7183d0af2e39078/C: closing flushed file at 1731522320580 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58f13fb7: reopening flushed file at 1731522320606 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b9f061a: reopening flushed file at 1731522320613 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39012283: reopening flushed file at 1731522320627 (+14 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 304012adc4150a48d7183d0af2e39078 in 626ms, sequenceid=693, compaction requested=true at 1731522320662 (+35 ms)Writing region close event to WAL at 1731522321426 (+764 ms)Running coprocessor post-close hooks at 1731522321433 (+7 ms)Closed at 1731522321433 2024-11-13T18:25:21,447 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed 304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:21,447 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=304012adc4150a48d7183d0af2e39078, regionState=CLOSED 2024-11-13T18:25:21,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:25:21,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-13T18:25:21,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure 304012adc4150a48d7183d0af2e39078, server=62ae331128c9,39615,1731522225114 in 1.5890 sec 2024-11-13T18:25:21,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-13T18:25:21,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=304012adc4150a48d7183d0af2e39078, UNASSIGN in 1.6090 sec 2024-11-13T18:25:21,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-13T18:25:21,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6210 sec 2024-11-13T18:25:21,481 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522321481"}]},"ts":"1731522321481"} 2024-11-13T18:25:21,496 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:25:21,496 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:25:21,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6680 sec 2024-11-13T18:25:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:25:22,023 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:22,027 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:25:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:22,035 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:22,039 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:25:22,100 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:22,105 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/recovered.edits] 2024-11-13T18:25:22,119 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/99c2f462ac5d4c7ca9911a2f99310bc5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/99c2f462ac5d4c7ca9911a2f99310bc5 2024-11-13T18:25:22,125 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/e0a1f363524a4b3bb162e6853211620f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/e0a1f363524a4b3bb162e6853211620f 2024-11-13T18:25:22,127 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fd080d0ee6d74ed1be793f7bf5a03258 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/fd080d0ee6d74ed1be793f7bf5a03258 2024-11-13T18:25:22,128 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/feb9e330fe214d308be6a30c7df9f5b3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/A/feb9e330fe214d308be6a30c7df9f5b3 2024-11-13T18:25:22,133 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/741c318f462a4ef283f6f299d0755340 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/741c318f462a4ef283f6f299d0755340 2024-11-13T18:25:22,134 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/9307d237c6fe47999a2ef7154db02feb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/9307d237c6fe47999a2ef7154db02feb 2024-11-13T18:25:22,139 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a277ee651f914096ab3f40a80fe8f216 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/a277ee651f914096ab3f40a80fe8f216 2024-11-13T18:25:22,141 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/c50968c07c5045a595e051d08022fdea to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/B/c50968c07c5045a595e051d08022fdea 2024-11-13T18:25:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:25:22,161 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/02eb1821c15145598e9a15d06a433910 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/02eb1821c15145598e9a15d06a433910 2024-11-13T18:25:22,169 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/652943b50941417ab7b09686a351c444 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/652943b50941417ab7b09686a351c444 2024-11-13T18:25:22,186 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/d800b0b0481d4d97bc9fcae9fa50106c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/d800b0b0481d4d97bc9fcae9fa50106c 2024-11-13T18:25:22,194 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/fa55cb32fa594fe4b82f6e75cab5c031 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/C/fa55cb32fa594fe4b82f6e75cab5c031 2024-11-13T18:25:22,206 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/recovered.edits/696.seqid to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078/recovered.edits/696.seqid 2024-11-13T18:25:22,206 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/304012adc4150a48d7183d0af2e39078 2024-11-13T18:25:22,213 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:25:22,230 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:22,236 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:25:22,240 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:25:22,242 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:22,242 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:25:22,242 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731522322242"}]},"ts":"9223372036854775807"} 2024-11-13T18:25:22,246 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:25:22,246 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 304012adc4150a48d7183d0af2e39078, NAME => 'TestAcidGuarantees,,1731522295940.304012adc4150a48d7183d0af2e39078.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:25:22,246 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:25:22,246 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731522322246"}]},"ts":"9223372036854775807"} 2024-11-13T18:25:22,249 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:25:22,250 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:22,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 223 msec 2024-11-13T18:25:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:25:22,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:25:22,374 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:22,392 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_633754790_22 at /127.0.0.1:56008 [Waiting for operation #332] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-536054504_22 at /127.0.0.1:55982 [Waiting for operation #340] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=2083 (was 2215), ProcessCount=11 (was 11), AvailableMemoryMB=1541 (was 1310) - AvailableMemoryMB LEAK? - 2024-11-13T18:25:22,407 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=2083, ProcessCount=11, AvailableMemoryMB=1540 2024-11-13T18:25:22,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:25:22,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:25:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:22,419 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:25:22,419 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:22,420 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-13T18:25:22,420 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:25:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-13T18:25:22,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742169_1345 (size=1061) 2024-11-13T18:25:22,495 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b 2024-11-13T18:25:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-13T18:25:22,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742170_1346 (size=53) 2024-11-13T18:25:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-13T18:25:22,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:22,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing aa8bc3911d985a299e88f859af353050, disabling compactions & flushes 2024-11-13T18:25:22,972 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:22,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:22,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. after waiting 0 ms 2024-11-13T18:25:22,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:22,972 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:22,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for aa8bc3911d985a299e88f859af353050: Waiting for close lock at 1731522322972Disabling compacts and flushes for region at 1731522322972Disabling writes for close at 1731522322972Writing region close event to WAL at 1731522322972Closed at 1731522322972 2024-11-13T18:25:22,979 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:25:22,979 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731522322979"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731522322979"}]},"ts":"1731522322979"} 2024-11-13T18:25:22,990 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:25:22,995 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:25:22,995 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522322995"}]},"ts":"1731522322995"} 2024-11-13T18:25:22,998 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:25:22,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, ASSIGN}] 2024-11-13T18:25:23,006 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, ASSIGN 2024-11-13T18:25:23,010 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, ASSIGN; state=OFFLINE, location=62ae331128c9,39615,1731522225114; forceNewPlan=false, retain=false 2024-11-13T18:25:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-13T18:25:23,162 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=aa8bc3911d985a299e88f859af353050, regionState=OPENING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:25:23,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, ASSIGN because future has completed 2024-11-13T18:25:23,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:25:23,328 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:23,329 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:23,329 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,329 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:23,329 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,329 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,342 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,352 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:25:23,352 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa8bc3911d985a299e88f859af353050 columnFamilyName A 2024-11-13T18:25:23,352 DEBUG [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:23,353 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(327): Store=aa8bc3911d985a299e88f859af353050/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:23,353 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,356 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:25:23,356 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa8bc3911d985a299e88f859af353050 columnFamilyName B 2024-11-13T18:25:23,357 DEBUG [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:23,363 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(327): Store=aa8bc3911d985a299e88f859af353050/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:23,363 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,369 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:25:23,369 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa8bc3911d985a299e88f859af353050 columnFamilyName C 2024-11-13T18:25:23,369 DEBUG [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:23,375 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(327): Store=aa8bc3911d985a299e88f859af353050/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:23,375 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:23,375 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,379 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,382 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,384 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,384 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,385 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:25:23,386 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,393 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:23,394 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened aa8bc3911d985a299e88f859af353050; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68354623, jitterRate=0.018563255667686462}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:25:23,394 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:23,395 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for aa8bc3911d985a299e88f859af353050: Running coprocessor pre-open hook at 1731522323329Writing region info on filesystem at 1731522323329Initializing all the Stores at 1731522323335 (+6 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522323335Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522323339 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522323339Cleaning up temporary data from old regions at 1731522323384 (+45 ms)Running coprocessor post-open hooks at 1731522323394 (+10 ms)Region opened successfully at 1731522323395 (+1 ms) 2024-11-13T18:25:23,397 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., pid=81, masterSystemTime=1731522323323 2024-11-13T18:25:23,406 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:23,406 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:23,407 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=aa8bc3911d985a299e88f859af353050, regionState=OPEN, openSeqNum=2, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:25:23,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:25:23,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-13T18:25:23,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 in 244 msec 2024-11-13T18:25:23,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-13T18:25:23,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, ASSIGN in 415 msec 2024-11-13T18:25:23,421 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:25:23,422 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522323421"}]},"ts":"1731522323421"} 2024-11-13T18:25:23,425 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:25:23,428 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:25:23,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.0140 sec 2024-11-13T18:25:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-13T18:25:23,586 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:23,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7696cee8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:23,587 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:23,587 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:23,589 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b4b5c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:23,590 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4d23a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:23,592 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:23,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:23,599 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:25:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 62ae331128c9,45483,1731522221514 2024-11-13T18:25:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@d9adc17 2024-11-13T18:25:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:25:23,602 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:25:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:25:23,605 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:25:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:23,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742171_1347 (size=1097) 2024-11-13T18:25:24,054 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T18:25:24,054 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T18:25:24,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:25:24,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, REOPEN/MOVE}] 2024-11-13T18:25:24,070 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, REOPEN/MOVE 2024-11-13T18:25:24,071 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=aa8bc3911d985a299e88f859af353050, regionState=CLOSING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:25:24,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, REOPEN/MOVE because future has completed 2024-11-13T18:25:24,075 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:25:24,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:25:24,233 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,233 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:25:24,233 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing aa8bc3911d985a299e88f859af353050, disabling compactions & flushes 2024-11-13T18:25:24,234 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:24,234 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:24,234 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. after waiting 0 ms 2024-11-13T18:25:24,234 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:24,286 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:25:24,295 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:24,295 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for aa8bc3911d985a299e88f859af353050: Waiting for close lock at 1731522324233Running coprocessor pre-close hooks at 1731522324233Disabling compacts and flushes for region at 1731522324233Disabling writes for close at 1731522324234 (+1 ms)Writing region close event to WAL at 1731522324235 (+1 ms)Running coprocessor post-close hooks at 1731522324295 (+60 ms)Closed at 1731522324295 2024-11-13T18:25:24,295 WARN [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: aa8bc3911d985a299e88f859af353050 to self. 2024-11-13T18:25:24,300 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=aa8bc3911d985a299e88f859af353050, regionState=CLOSED 2024-11-13T18:25:24,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:25:24,306 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-13T18:25:24,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 in 243 msec 2024-11-13T18:25:24,321 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, REOPEN/MOVE; state=CLOSED, location=62ae331128c9,39615,1731522225114; forceNewPlan=false, retain=true 2024-11-13T18:25:24,472 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=aa8bc3911d985a299e88f859af353050, regionState=OPENING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:25:24,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, REOPEN/MOVE because future has completed 2024-11-13T18:25:24,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:25:24,645 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:24,645 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:24,645 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,645 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:24,646 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,646 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,660 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,662 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:25:24,663 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa8bc3911d985a299e88f859af353050 columnFamilyName A 2024-11-13T18:25:24,665 DEBUG [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:24,666 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(327): Store=aa8bc3911d985a299e88f859af353050/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:24,667 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,668 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:25:24,668 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa8bc3911d985a299e88f859af353050 columnFamilyName B 2024-11-13T18:25:24,668 DEBUG [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:24,669 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(327): Store=aa8bc3911d985a299e88f859af353050/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:24,669 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,669 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:25:24,670 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa8bc3911d985a299e88f859af353050 columnFamilyName C 2024-11-13T18:25:24,670 DEBUG [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:24,670 INFO [StoreOpener-aa8bc3911d985a299e88f859af353050-1 {}] regionserver.HStore(327): Store=aa8bc3911d985a299e88f859af353050/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:24,670 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:24,671 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,672 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,673 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,675 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,675 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,676 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:25:24,678 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,680 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened aa8bc3911d985a299e88f859af353050; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71233590, jitterRate=0.06146320700645447}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:25:24,680 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,681 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for aa8bc3911d985a299e88f859af353050: Running coprocessor pre-open hook at 1731522324646Writing region info on filesystem at 1731522324646Initializing all the Stores at 1731522324648 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522324648Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522324649 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522324649Cleaning up temporary data from old regions at 1731522324675 (+26 ms)Running coprocessor post-open hooks at 1731522324680 (+5 ms)Region opened successfully at 1731522324681 (+1 ms) 2024-11-13T18:25:24,682 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., pid=86, masterSystemTime=1731522324630 2024-11-13T18:25:24,687 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:24,687 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:24,688 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=aa8bc3911d985a299e88f859af353050, regionState=OPEN, openSeqNum=5, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:25:24,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:25:24,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-13T18:25:24,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 in 222 msec 2024-11-13T18:25:24,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-13T18:25:24,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, REOPEN/MOVE in 635 msec 2024-11-13T18:25:24,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-13T18:25:24,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 654 msec 2024-11-13T18:25:24,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.1130 sec 2024-11-13T18:25:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T18:25:24,724 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:24,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c8520b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:24,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:24,727 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23760892, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:24,732 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c942a42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a69c70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,739 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:24,739 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:24,742 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:24,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:24,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad7a947, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:24,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:24,747 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192ed6a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1398ece7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:24,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:24,771 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:24,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25671080, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:24,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:24,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:24,780 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:24,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c41d341, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd9f4fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:24,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:24,786 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5314b2ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:24,791 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e150c2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36bba6b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:24,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:24,804 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a1fa06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:24,808 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:24,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eaf7439, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71e94d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:24,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:24,820 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ba5c66b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:24,823 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6716d20c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2efff8d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:24,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:24,841 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256f7233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:24,845 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e82f58c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f5bbd72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:24,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:24,869 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b1e48e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:24,874 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c740dee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@141e8cce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:24,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:24,881 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33083dcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:24,884 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4da43bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21fc0fcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:24,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:24,892 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae71194, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:24,895 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a8aa470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:24,899 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-13T18:25:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T18:25:24,902 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:24,903 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:24,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:24,911 DEBUG [Thread-1593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:24,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:25:24,931 DEBUG [Thread-1597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:24,934 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:24,935 DEBUG [Thread-1584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:24,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:24,935 DEBUG [Thread-1590 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:24,935 DEBUG [Thread-1584 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:24,936 DEBUG [Thread-1590 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:24,936 DEBUG [Thread-1584 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:24,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:25:24,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:24,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:24,939 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:25:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:25:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:24,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:25:24,962 DEBUG [Thread-1582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:24,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:24,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:24,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:24,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:24,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:24,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:25:24,971 DEBUG [Thread-1595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:24,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:25:24,991 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:24,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:24,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:25:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T18:25:25,018 DEBUG [Thread-1586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:25,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:25,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:25:25,035 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:25,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:25:25,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:25,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133bd54c6d98804b8db0d67d5830326dee_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522324945/Put/seqid=0 2024-11-13T18:25:25,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731522325001, max timestamp=1731522325094Num uniques 859; 2024-11-13T18:25:25,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:25:25,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731522325001, max timestamp=1731522325094Num uniques 859; 2024-11-13T18:25:25,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:25:25,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731522325001, max timestamp=1731522325094Num uniques 859; 2024-11-13T18:25:25,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T18:25:25,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742172_1348 (size=12154) 2024-11-13T18:25:25,158 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:25,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133bd54c6d98804b8db0d67d5830326dee_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133bd54c6d98804b8db0d67d5830326dee_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:25,182 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/269bf68e36594331a02ab80e193687f5, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:25,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/269bf68e36594331a02ab80e193687f5 is 175, key is test_row_0/A:col10/1731522324945/Put/seqid=0 2024-11-13T18:25:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T18:25:25,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742173_1349 (size=30955) 2024-11-13T18:25:25,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522385214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522385215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522385223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522385223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522385226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522385346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522385346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522385346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522385356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522385357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:25,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:25,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T18:25:25,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:25,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:25,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522385566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522385566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522385567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522385577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522385577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,634 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/269bf68e36594331a02ab80e193687f5 2024-11-13T18:25:25,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:25,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:25,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/790194f8f26c4514bebd0af5d7d67889 is 50, key is test_row_0/B:col10/1731522324945/Put/seqid=0 2024-11-13T18:25:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742174_1350 (size=12001) 2024-11-13T18:25:25,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:25,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:25,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:25,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:25,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522385906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522385906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522385907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522385907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522385907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:26,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:26,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:26,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:26,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:26,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T18:25:26,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/790194f8f26c4514bebd0af5d7d67889 2024-11-13T18:25:26,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:26,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/01b6e654edf6436abd310df538944588 is 50, key is test_row_0/C:col10/1731522324945/Put/seqid=0 2024-11-13T18:25:26,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742175_1351 (size=12001) 2024-11-13T18:25:26,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:26,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/01b6e654edf6436abd310df538944588 2024-11-13T18:25:26,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/269bf68e36594331a02ab80e193687f5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/269bf68e36594331a02ab80e193687f5 2024-11-13T18:25:26,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/269bf68e36594331a02ab80e193687f5, entries=150, sequenceid=14, filesize=30.2 K 2024-11-13T18:25:26,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/790194f8f26c4514bebd0af5d7d67889 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/790194f8f26c4514bebd0af5d7d67889 2024-11-13T18:25:26,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/790194f8f26c4514bebd0af5d7d67889, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T18:25:26,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/01b6e654edf6436abd310df538944588 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01b6e654edf6436abd310df538944588 2024-11-13T18:25:26,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522386448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522386448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:26,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522386448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:26,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522386449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:26,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:26,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:26,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:26,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:26,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:26,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522386448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:26,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01b6e654edf6436abd310df538944588, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T18:25:26,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:26,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:26,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:26,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:26,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for aa8bc3911d985a299e88f859af353050 in 1546ms, sequenceid=14, compaction requested=false 2024-11-13T18:25:26,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,510 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T18:25:26,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:26,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T18:25:26,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:26,665 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-13T18:25:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136425068a3ae94c489e37b8de8ff828ec_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522325218/Put/seqid=0 2024-11-13T18:25:26,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742176_1352 (size=12154) 2024-11-13T18:25:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T18:25:27,064 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:25:27,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:27,178 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136425068a3ae94c489e37b8de8ff828ec_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136425068a3ae94c489e37b8de8ff828ec_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/b41e6e51caed4625ac67f459379d40e8, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/b41e6e51caed4625ac67f459379d40e8 is 175, key is test_row_0/A:col10/1731522325218/Put/seqid=0 2024-11-13T18:25:27,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742177_1353 (size=30955) 2024-11-13T18:25:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:27,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522387485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522387509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522387509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522387518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522387519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,635 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/b41e6e51caed4625ac67f459379d40e8 2024-11-13T18:25:27,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522387632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522387646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522387646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522387646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522387646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/f58e43ee1782442196c17b0ae48aa832 is 50, key is test_row_0/B:col10/1731522325218/Put/seqid=0 2024-11-13T18:25:27,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742178_1354 (size=12001) 2024-11-13T18:25:27,730 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/f58e43ee1782442196c17b0ae48aa832 2024-11-13T18:25:27,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/3d94a76624034349b5e226d9303c0789 is 50, key is test_row_0/C:col10/1731522325218/Put/seqid=0 2024-11-13T18:25:27,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742179_1355 (size=12001) 2024-11-13T18:25:27,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522387866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522387867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522387867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522387867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522387867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:28,199 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/3d94a76624034349b5e226d9303c0789 2024-11-13T18:25:28,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522388187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:28,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/b41e6e51caed4625ac67f459379d40e8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/b41e6e51caed4625ac67f459379d40e8 2024-11-13T18:25:28,222 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/b41e6e51caed4625ac67f459379d40e8, entries=150, sequenceid=51, filesize=30.2 K 2024-11-13T18:25:28,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/f58e43ee1782442196c17b0ae48aa832 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f58e43ee1782442196c17b0ae48aa832 2024-11-13T18:25:28,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522388206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:28,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:28,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522388206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:28,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522388206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:28,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:28,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522388206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:28,231 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f58e43ee1782442196c17b0ae48aa832, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T18:25:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/3d94a76624034349b5e226d9303c0789 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/3d94a76624034349b5e226d9303c0789 2024-11-13T18:25:28,244 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/3d94a76624034349b5e226d9303c0789, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T18:25:28,245 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=40.25 KB/41220 for aa8bc3911d985a299e88f859af353050 in 1580ms, sequenceid=51, compaction requested=false 2024-11-13T18:25:28,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:28,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:28,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-13T18:25:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-13T18:25:28,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-13T18:25:28,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3440 sec 2024-11-13T18:25:28,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 3.3520 sec 2024-11-13T18:25:28,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:28,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:28,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:28,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:28,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:28,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:28,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:28,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136b2bbc7aa6cc430da0d71d64d9e9212e_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522327509/Put/seqid=0 2024-11-13T18:25:28,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742180_1356 (size=12154) 2024-11-13T18:25:28,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731522328813, max timestamp=1731522328868Num uniques 859; 2024-11-13T18:25:28,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:25:28,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731522328813, max timestamp=1731522328868Num uniques 859; 2024-11-13T18:25:28,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731522328813, max timestamp=1731522328868Num uniques 859; 2024-11-13T18:25:28,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:25:28,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T18:25:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522388970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522388975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522388996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522388998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522389001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T18:25:29,076 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:29,076 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-13T18:25:29,078 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:29,080 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:29,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T18:25:29,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522389106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522389106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522389107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522389128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522389129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T18:25:29,212 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:29,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136b2bbc7aa6cc430da0d71d64d9e9212e_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136b2bbc7aa6cc430da0d71d64d9e9212e_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:29,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/a785d518751d48f194a22cf2be893bdc, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:29,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/a785d518751d48f194a22cf2be893bdc is 175, key is test_row_0/A:col10/1731522327509/Put/seqid=0 2024-11-13T18:25:29,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:29,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742181_1357 (size=30955) 2024-11-13T18:25:29,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522389326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522389326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522389326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522389346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522389346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:29,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:29,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:29,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:29,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T18:25:29,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:29,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:29,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:29,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:29,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,670 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/a785d518751d48f194a22cf2be893bdc 2024-11-13T18:25:29,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522389659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522389660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522389660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522389667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522389667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:29,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/2521a33dfa50474a862302c3053b645f is 50, key is test_row_0/B:col10/1731522327509/Put/seqid=0 2024-11-13T18:25:29,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:29,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:29,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:29,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:29,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T18:25:29,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742182_1358 (size=12001) 2024-11-13T18:25:29,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:29,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:29,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/2521a33dfa50474a862302c3053b645f 2024-11-13T18:25:30,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/a878e8e218d34e34993a5bedb0d17866 is 50, key is test_row_0/C:col10/1731522327509/Put/seqid=0 2024-11-13T18:25:30,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:30,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:30,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742183_1359 (size=12001) 2024-11-13T18:25:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522390187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:30,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:30,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:30,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:30,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522390187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:30,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522390187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:30,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522390188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:30,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522390196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:30,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T18:25:30,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:30,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:30,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/a878e8e218d34e34993a5bedb0d17866 2024-11-13T18:25:30,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/a785d518751d48f194a22cf2be893bdc as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/a785d518751d48f194a22cf2be893bdc 2024-11-13T18:25:30,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/a785d518751d48f194a22cf2be893bdc, entries=150, sequenceid=61, filesize=30.2 K 2024-11-13T18:25:30,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/2521a33dfa50474a862302c3053b645f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/2521a33dfa50474a862302c3053b645f 2024-11-13T18:25:30,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/2521a33dfa50474a862302c3053b645f, entries=150, sequenceid=61, filesize=11.7 K 2024-11-13T18:25:30,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/a878e8e218d34e34993a5bedb0d17866 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/a878e8e218d34e34993a5bedb0d17866 2024-11-13T18:25:30,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:30,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:30,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:30,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/a878e8e218d34e34993a5bedb0d17866, entries=150, sequenceid=61, filesize=11.7 K 2024-11-13T18:25:30,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for aa8bc3911d985a299e88f859af353050 in 1930ms, sequenceid=61, compaction requested=true 2024-11-13T18:25:30,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:30,664 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:30,665 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:30,665 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/A is initiating minor compaction (all files) 2024-11-13T18:25:30,665 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/A in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,665 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/269bf68e36594331a02ab80e193687f5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/b41e6e51caed4625ac67f459379d40e8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/a785d518751d48f194a22cf2be893bdc] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=90.7 K 2024-11-13T18:25:30,665 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,666 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/269bf68e36594331a02ab80e193687f5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/b41e6e51caed4625ac67f459379d40e8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/a785d518751d48f194a22cf2be893bdc] 2024-11-13T18:25:30,666 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 269bf68e36594331a02ab80e193687f5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731522324945 2024-11-13T18:25:30,666 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b41e6e51caed4625ac67f459379d40e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731522325201 2024-11-13T18:25:30,666 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting a785d518751d48f194a22cf2be893bdc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731522327477 2024-11-13T18:25:30,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:30,678 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:30,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:30,679 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:30,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:30,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:30,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:30,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:30,680 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:30,680 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/B is initiating minor compaction (all files) 2024-11-13T18:25:30,680 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/B in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,681 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/790194f8f26c4514bebd0af5d7d67889, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f58e43ee1782442196c17b0ae48aa832, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/2521a33dfa50474a862302c3053b645f] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=35.2 K 2024-11-13T18:25:30,681 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 790194f8f26c4514bebd0af5d7d67889, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731522324945 2024-11-13T18:25:30,682 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting f58e43ee1782442196c17b0ae48aa832, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731522325201 2024-11-13T18:25:30,682 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 2521a33dfa50474a862302c3053b645f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731522327477 2024-11-13T18:25:30,701 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113d7cde0d0c2da450ab5a85bc32baddacd_aa8bc3911d985a299e88f859af353050 store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:30,704 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113d7cde0d0c2da450ab5a85bc32baddacd_aa8bc3911d985a299e88f859af353050, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:30,704 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d7cde0d0c2da450ab5a85bc32baddacd_aa8bc3911d985a299e88f859af353050 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:30,704 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:30,705 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/f612e2cca7764aa1926f603cb6bea22f is 50, key is test_row_0/B:col10/1731522327509/Put/seqid=0 2024-11-13T18:25:30,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742184_1360 (size=4469) 2024-11-13T18:25:30,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742185_1361 (size=12104) 2024-11-13T18:25:30,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T18:25:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:30,805 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-13T18:25:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:30,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:30,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e0e9d8fc9f014534b1af9e2639ea072b_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522328998/Put/seqid=0 2024-11-13T18:25:30,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742186_1362 (size=12154) 2024-11-13T18:25:30,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:30,866 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e0e9d8fc9f014534b1af9e2639ea072b_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e0e9d8fc9f014534b1af9e2639ea072b_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:30,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/2a2407a1e51147af9cede586c366317a, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:30,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/2a2407a1e51147af9cede586c366317a is 175, key is test_row_0/A:col10/1731522328998/Put/seqid=0 2024-11-13T18:25:30,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742187_1363 (size=30955) 2024-11-13T18:25:30,905 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/2a2407a1e51147af9cede586c366317a 2024-11-13T18:25:30,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/9240a5d8d6774add90f8a47d4a243773 is 50, key is test_row_0/B:col10/1731522328998/Put/seqid=0 2024-11-13T18:25:30,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742188_1364 (size=12001) 2024-11-13T18:25:31,153 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#A#compaction#306 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:31,153 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/91b4d300aaed434da024c12f395c33eb is 175, key is test_row_0/A:col10/1731522327509/Put/seqid=0 2024-11-13T18:25:31,161 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/f612e2cca7764aa1926f603cb6bea22f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f612e2cca7764aa1926f603cb6bea22f 2024-11-13T18:25:31,169 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/B of aa8bc3911d985a299e88f859af353050 into f612e2cca7764aa1926f603cb6bea22f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:31,169 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:31,169 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/B, priority=13, startTime=1731522330678; duration=0sec 2024-11-13T18:25:31,169 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:31,169 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:B 2024-11-13T18:25:31,169 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:31,172 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:31,172 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/C is initiating minor compaction (all files) 2024-11-13T18:25:31,172 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/C in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:31,172 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01b6e654edf6436abd310df538944588, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/3d94a76624034349b5e226d9303c0789, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/a878e8e218d34e34993a5bedb0d17866] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=35.2 K 2024-11-13T18:25:31,173 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 01b6e654edf6436abd310df538944588, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731522324945 2024-11-13T18:25:31,174 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d94a76624034349b5e226d9303c0789, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731522325201 2024-11-13T18:25:31,174 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting a878e8e218d34e34993a5bedb0d17866, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731522327477 2024-11-13T18:25:31,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742189_1365 (size=31058) 2024-11-13T18:25:31,213 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#C#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:31,215 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/89b82b471e3e4276b102b8e791a7a2d6 is 50, key is test_row_0/C:col10/1731522327509/Put/seqid=0 2024-11-13T18:25:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T18:25:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:31,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:31,275 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/91b4d300aaed434da024c12f395c33eb as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/91b4d300aaed434da024c12f395c33eb 2024-11-13T18:25:31,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742190_1366 (size=12104) 2024-11-13T18:25:31,317 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/89b82b471e3e4276b102b8e791a7a2d6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/89b82b471e3e4276b102b8e791a7a2d6 2024-11-13T18:25:31,319 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/A of aa8bc3911d985a299e88f859af353050 into 91b4d300aaed434da024c12f395c33eb(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:31,319 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:31,319 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/A, priority=13, startTime=1731522330664; duration=0sec 2024-11-13T18:25:31,319 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:31,319 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:A 2024-11-13T18:25:31,336 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/C of aa8bc3911d985a299e88f859af353050 into 89b82b471e3e4276b102b8e791a7a2d6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:31,336 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:31,336 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/C, priority=13, startTime=1731522330680; duration=0sec 2024-11-13T18:25:31,336 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:31,336 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:C 2024-11-13T18:25:31,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522391321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522391322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522391323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522391324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,371 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/9240a5d8d6774add90f8a47d4a243773 2024-11-13T18:25:31,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522391354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/268e0d0b140a4e32aef185313466593e is 50, key is test_row_0/C:col10/1731522328998/Put/seqid=0 2024-11-13T18:25:31,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742191_1367 (size=12001) 2024-11-13T18:25:31,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522391466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522391467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522391467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522391467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522391476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522391698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522391698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522391698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522391698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522391698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:31,877 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/268e0d0b140a4e32aef185313466593e 2024-11-13T18:25:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/2a2407a1e51147af9cede586c366317a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/2a2407a1e51147af9cede586c366317a 2024-11-13T18:25:31,907 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/2a2407a1e51147af9cede586c366317a, entries=150, sequenceid=96, filesize=30.2 K 2024-11-13T18:25:31,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/9240a5d8d6774add90f8a47d4a243773 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/9240a5d8d6774add90f8a47d4a243773 2024-11-13T18:25:31,937 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/9240a5d8d6774add90f8a47d4a243773, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T18:25:31,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/268e0d0b140a4e32aef185313466593e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/268e0d0b140a4e32aef185313466593e 2024-11-13T18:25:31,962 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/268e0d0b140a4e32aef185313466593e, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T18:25:31,967 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for aa8bc3911d985a299e88f859af353050 in 1162ms, sequenceid=96, compaction requested=false 2024-11-13T18:25:31,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:31,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:31,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-13T18:25:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-13T18:25:31,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-13T18:25:31,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8990 sec 2024-11-13T18:25:31,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.9060 sec 2024-11-13T18:25:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:32,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:25:32,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:32,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:32,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:32,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:32,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:32,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:32,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113038df1d3cce2483fad3b358a9c1388f7_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522332073/Put/seqid=0 2024-11-13T18:25:32,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742192_1368 (size=14594) 2024-11-13T18:25:32,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522392142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522392146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522392156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522392163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522392167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522392283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522392283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522392286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522392295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522392307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522392496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522392516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522392516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522392516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522392539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,554 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:32,562 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113038df1d3cce2483fad3b358a9c1388f7_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113038df1d3cce2483fad3b358a9c1388f7_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:32,563 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/1ee76691050641de91a76bdfa96bbbe7, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:32,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/1ee76691050641de91a76bdfa96bbbe7 is 175, key is test_row_0/A:col10/1731522332073/Put/seqid=0 2024-11-13T18:25:32,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742193_1369 (size=39549) 2024-11-13T18:25:32,605 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/1ee76691050641de91a76bdfa96bbbe7 2024-11-13T18:25:32,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e24c0ba1fa9d4b81893756695bc8668e is 50, key is test_row_0/B:col10/1731522332073/Put/seqid=0 2024-11-13T18:25:32,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742194_1370 (size=12001) 2024-11-13T18:25:32,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e24c0ba1fa9d4b81893756695bc8668e 2024-11-13T18:25:32,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/dc8f3bb03207418fb082550d7043c53f is 50, key is test_row_0/C:col10/1731522332073/Put/seqid=0 2024-11-13T18:25:32,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742195_1371 (size=12001) 2024-11-13T18:25:32,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522392815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522392858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522392858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522392860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522392860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/dc8f3bb03207418fb082550d7043c53f 2024-11-13T18:25:33,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/1ee76691050641de91a76bdfa96bbbe7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/1ee76691050641de91a76bdfa96bbbe7 2024-11-13T18:25:33,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/1ee76691050641de91a76bdfa96bbbe7, entries=200, sequenceid=113, filesize=38.6 K 2024-11-13T18:25:33,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e24c0ba1fa9d4b81893756695bc8668e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e24c0ba1fa9d4b81893756695bc8668e 2024-11-13T18:25:33,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e24c0ba1fa9d4b81893756695bc8668e, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T18:25:33,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/dc8f3bb03207418fb082550d7043c53f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/dc8f3bb03207418fb082550d7043c53f 2024-11-13T18:25:33,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T18:25:33,266 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:33,268 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:33,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/dc8f3bb03207418fb082550d7043c53f, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T18:25:33,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for aa8bc3911d985a299e88f859af353050 in 1195ms, sequenceid=113, compaction requested=true 2024-11-13T18:25:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:33,270 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:33,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:33,271 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:33,272 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:33,272 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/A is initiating minor compaction (all files) 2024-11-13T18:25:33,272 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/A in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,272 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/91b4d300aaed434da024c12f395c33eb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/2a2407a1e51147af9cede586c366317a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/1ee76691050641de91a76bdfa96bbbe7] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=99.2 K 2024-11-13T18:25:33,272 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,272 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/91b4d300aaed434da024c12f395c33eb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/2a2407a1e51147af9cede586c366317a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/1ee76691050641de91a76bdfa96bbbe7] 2024-11-13T18:25:33,272 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:33,272 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/B is initiating minor compaction (all files) 2024-11-13T18:25:33,272 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/B in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,273 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f612e2cca7764aa1926f603cb6bea22f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/9240a5d8d6774add90f8a47d4a243773, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e24c0ba1fa9d4b81893756695bc8668e] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=35.3 K 2024-11-13T18:25:33,273 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting f612e2cca7764aa1926f603cb6bea22f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731522327477 2024-11-13T18:25:33,273 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91b4d300aaed434da024c12f395c33eb, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731522327477 2024-11-13T18:25:33,273 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9240a5d8d6774add90f8a47d4a243773, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731522328948 2024-11-13T18:25:33,273 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a2407a1e51147af9cede586c366317a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731522328948 2024-11-13T18:25:33,274 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e24c0ba1fa9d4b81893756695bc8668e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731522331316 2024-11-13T18:25:33,274 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ee76691050641de91a76bdfa96bbbe7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731522331316 2024-11-13T18:25:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-13T18:25:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:25:33,285 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:33,286 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/657693235d4846f0976c4e152c2120d4 is 50, key is test_row_0/B:col10/1731522332073/Put/seqid=0 2024-11-13T18:25:33,286 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:33,287 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:33,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:33,288 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:33,302 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411136388d3fa5e3e43afa1e65643d52da5c4_aa8bc3911d985a299e88f859af353050 store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:33,304 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411136388d3fa5e3e43afa1e65643d52da5c4_aa8bc3911d985a299e88f859af353050, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:33,304 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136388d3fa5e3e43afa1e65643d52da5c4_aa8bc3911d985a299e88f859af353050 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:33,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:25:33,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:33,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:33,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:33,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:33,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:33,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:33,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742197_1373 (size=4469) 2024-11-13T18:25:33,380 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#A#compaction#316 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:33,381 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/51c4c78bf0ad432bb1a371aa6a1b161f is 175, key is test_row_0/A:col10/1731522332073/Put/seqid=0 2024-11-13T18:25:33,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742196_1372 (size=12207) 2024-11-13T18:25:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:25:33,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a71e1d775bda43f0b4b31c9a51fd64f8_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522332161/Put/seqid=0 2024-11-13T18:25:33,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742198_1374 (size=31161) 2024-11-13T18:25:33,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:25:33,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:33,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522393418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522393424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,450 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/51c4c78bf0ad432bb1a371aa6a1b161f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/51c4c78bf0ad432bb1a371aa6a1b161f 2024-11-13T18:25:33,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522393430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,461 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/A of aa8bc3911d985a299e88f859af353050 into 51c4c78bf0ad432bb1a371aa6a1b161f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:33,461 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:33,461 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/A, priority=13, startTime=1731522333270; duration=0sec 2024-11-13T18:25:33,461 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:33,461 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:A 2024-11-13T18:25:33,461 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:33,466 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:33,467 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/C is initiating minor compaction (all files) 2024-11-13T18:25:33,467 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/C in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742199_1375 (size=14744) 2024-11-13T18:25:33,467 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/89b82b471e3e4276b102b8e791a7a2d6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/268e0d0b140a4e32aef185313466593e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/dc8f3bb03207418fb082550d7043c53f] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=35.3 K 2024-11-13T18:25:33,470 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89b82b471e3e4276b102b8e791a7a2d6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731522327477 2024-11-13T18:25:33,470 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 268e0d0b140a4e32aef185313466593e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731522328948 2024-11-13T18:25:33,470 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc8f3bb03207418fb082550d7043c53f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731522331316 2024-11-13T18:25:33,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522393437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522393443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,490 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:33,491 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/6a2fcc74356f43b3a21c4a6b0e5895a9 is 50, key is test_row_0/C:col10/1731522332073/Put/seqid=0 2024-11-13T18:25:33,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742200_1376 (size=12207) 2024-11-13T18:25:33,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522393559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522393559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522393565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522393578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:25:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:25:33,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522393587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:25:33,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:33,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,796 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/657693235d4846f0976c4e152c2120d4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/657693235d4846f0976c4e152c2120d4 2024-11-13T18:25:33,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522393778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522393778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522393787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,811 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/B of aa8bc3911d985a299e88f859af353050 into 657693235d4846f0976c4e152c2120d4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:33,812 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:33,812 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/B, priority=13, startTime=1731522333270; duration=0sec 2024-11-13T18:25:33,812 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:33,812 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:B 2024-11-13T18:25:33,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522393796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522393818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:33,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:33,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:33,868 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:33,873 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a71e1d775bda43f0b4b31c9a51fd64f8_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a71e1d775bda43f0b4b31c9a51fd64f8_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:33,874 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/c53bad85e2f046cfbff89897716c358c, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:33,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/c53bad85e2f046cfbff89897716c358c is 175, key is test_row_0/A:col10/1731522332161/Put/seqid=0 2024-11-13T18:25:33,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:25:33,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:33,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:33,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:33,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742201_1377 (size=39699) 2024-11-13T18:25:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:25:33,921 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/c53bad85e2f046cfbff89897716c358c 2024-11-13T18:25:33,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/4c41580a4bfa4f4487870ae502005053 is 50, key is test_row_0/B:col10/1731522332161/Put/seqid=0 2024-11-13T18:25:33,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742202_1378 (size=12151) 2024-11-13T18:25:33,981 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/6a2fcc74356f43b3a21c4a6b0e5895a9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/6a2fcc74356f43b3a21c4a6b0e5895a9 2024-11-13T18:25:33,990 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/C of aa8bc3911d985a299e88f859af353050 into 6a2fcc74356f43b3a21c4a6b0e5895a9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:33,990 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:33,990 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/C, priority=13, startTime=1731522333270; duration=0sec 2024-11-13T18:25:33,990 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:33,990 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:C 2024-11-13T18:25:34,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:25:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:34,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:34,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522394106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:34,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522394106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:34,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:34,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522394107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:34,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522394138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:34,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522394157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:34,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:25:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/4c41580a4bfa4f4487870ae502005053 2024-11-13T18:25:34,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:25:34,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:34,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/baa29936c4784a88966bff6332608f19 is 50, key is test_row_0/C:col10/1731522332161/Put/seqid=0 2024-11-13T18:25:34,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742203_1379 (size=12151) 2024-11-13T18:25:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:25:34,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:25:34,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:34,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522394627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:34,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522394627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:34,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:34,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522394627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:34,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:34,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:34,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522394656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:34,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522394675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:34,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:25:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:34,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/baa29936c4784a88966bff6332608f19 2024-11-13T18:25:34,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/c53bad85e2f046cfbff89897716c358c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/c53bad85e2f046cfbff89897716c358c 2024-11-13T18:25:34,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/c53bad85e2f046cfbff89897716c358c, entries=200, sequenceid=132, filesize=38.8 K 2024-11-13T18:25:34,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/4c41580a4bfa4f4487870ae502005053 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4c41580a4bfa4f4487870ae502005053 2024-11-13T18:25:34,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4c41580a4bfa4f4487870ae502005053, entries=150, sequenceid=132, filesize=11.9 K 2024-11-13T18:25:34,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/baa29936c4784a88966bff6332608f19 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/baa29936c4784a88966bff6332608f19 2024-11-13T18:25:34,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/baa29936c4784a88966bff6332608f19, entries=150, sequenceid=132, filesize=11.9 K 2024-11-13T18:25:34,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for aa8bc3911d985a299e88f859af353050 in 1485ms, sequenceid=132, compaction requested=false 2024-11-13T18:25:34,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:34,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T18:25:34,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:34,862 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:25:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:34,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138dc2eac80ff84221b82e5deef8f068b6_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522333441/Put/seqid=0 2024-11-13T18:25:34,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742204_1380 (size=12304) 2024-11-13T18:25:35,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:35,313 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138dc2eac80ff84221b82e5deef8f068b6_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138dc2eac80ff84221b82e5deef8f068b6_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:35,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/bacf4726792d43acbee1d761758b1ec9, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:35,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/bacf4726792d43acbee1d761758b1ec9 is 175, key is test_row_0/A:col10/1731522333441/Put/seqid=0 2024-11-13T18:25:35,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742205_1381 (size=31105) 2024-11-13T18:25:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:25:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:35,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:35,727 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/bacf4726792d43acbee1d761758b1ec9 2024-11-13T18:25:35,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522395716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:35,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522395718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:35,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522395720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522395720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522395723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:35,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e6128e7a18a44140b3cec4e787f14458 is 50, key is test_row_0/B:col10/1731522333441/Put/seqid=0 2024-11-13T18:25:35,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742206_1382 (size=12151) 2024-11-13T18:25:35,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522395847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:35,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522395847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522395847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:35,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522395848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:35,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522395849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522396056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522396066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522396066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522396066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522396066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,157 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e6128e7a18a44140b3cec4e787f14458 2024-11-13T18:25:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/93437ae6e2ca48c18073f379d7a0e366 is 50, key is test_row_0/C:col10/1731522333441/Put/seqid=0 2024-11-13T18:25:36,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742207_1383 (size=12151) 2024-11-13T18:25:36,206 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/93437ae6e2ca48c18073f379d7a0e366 2024-11-13T18:25:36,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/bacf4726792d43acbee1d761758b1ec9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/bacf4726792d43acbee1d761758b1ec9 2024-11-13T18:25:36,226 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/bacf4726792d43acbee1d761758b1ec9, entries=150, sequenceid=148, filesize=30.4 K 2024-11-13T18:25:36,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e6128e7a18a44140b3cec4e787f14458 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e6128e7a18a44140b3cec4e787f14458 2024-11-13T18:25:36,235 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e6128e7a18a44140b3cec4e787f14458, entries=150, sequenceid=148, filesize=11.9 K 2024-11-13T18:25:36,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/93437ae6e2ca48c18073f379d7a0e366 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/93437ae6e2ca48c18073f379d7a0e366 2024-11-13T18:25:36,246 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/93437ae6e2ca48c18073f379d7a0e366, entries=150, sequenceid=148, filesize=11.9 K 2024-11-13T18:25:36,249 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for aa8bc3911d985a299e88f859af353050 in 1388ms, sequenceid=148, compaction requested=true 2024-11-13T18:25:36,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:36,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:36,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-13T18:25:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-13T18:25:36,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-13T18:25:36,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9630 sec 2024-11-13T18:25:36,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.9860 sec 2024-11-13T18:25:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:36,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:25:36,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:36,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:36,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:36,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:36,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:36,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:36,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e2736d9b35164b269a6e5ce24ff4ea59_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522335696/Put/seqid=0 2024-11-13T18:25:36,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522396404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522396406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522396407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522396409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522396410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742208_1384 (size=14794) 2024-11-13T18:25:36,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522396525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522396525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522396526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522396526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522396526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522396737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522396737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522396745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522396745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522396746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:36,834 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:36,838 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e2736d9b35164b269a6e5ce24ff4ea59_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e2736d9b35164b269a6e5ce24ff4ea59_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:36,840 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/72c4e1289b644799b96e75136615d055, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:36,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/72c4e1289b644799b96e75136615d055 is 175, key is test_row_0/A:col10/1731522335696/Put/seqid=0 2024-11-13T18:25:36,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742209_1385 (size=39749) 2024-11-13T18:25:37,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522397048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:37,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522397055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:37,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522397065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522397068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522397069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:37,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/72c4e1289b644799b96e75136615d055 2024-11-13T18:25:37,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/543abc6128704e2ba65a98794a8635ac is 50, key is test_row_0/B:col10/1731522335696/Put/seqid=0 2024-11-13T18:25:37,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742210_1386 (size=12151) 2024-11-13T18:25:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T18:25:37,467 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:37,468 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-13T18:25:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:25:37,476 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:37,478 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:37,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:37,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522397567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:25:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:37,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522397576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:37,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522397585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:37,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522397600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:37,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522397600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:37,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:37,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:25:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:37,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:37,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:37,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/543abc6128704e2ba65a98794a8635ac 2024-11-13T18:25:37,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/8bdb079cd04c43bdac5efb72003b8c12 is 50, key is test_row_0/C:col10/1731522335696/Put/seqid=0 2024-11-13T18:25:37,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742211_1387 (size=12151) 2024-11-13T18:25:37,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/8bdb079cd04c43bdac5efb72003b8c12 2024-11-13T18:25:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:25:37,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:25:37,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:37,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:37,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:37,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:37,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:37,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/72c4e1289b644799b96e75136615d055 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/72c4e1289b644799b96e75136615d055 2024-11-13T18:25:37,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/72c4e1289b644799b96e75136615d055, entries=200, sequenceid=168, filesize=38.8 K 2024-11-13T18:25:37,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/543abc6128704e2ba65a98794a8635ac as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/543abc6128704e2ba65a98794a8635ac 2024-11-13T18:25:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/543abc6128704e2ba65a98794a8635ac, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T18:25:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/8bdb079cd04c43bdac5efb72003b8c12 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8bdb079cd04c43bdac5efb72003b8c12 2024-11-13T18:25:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8bdb079cd04c43bdac5efb72003b8c12, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T18:25:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for aa8bc3911d985a299e88f859af353050 in 1456ms, sequenceid=168, compaction requested=true 2024-11-13T18:25:37,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:37,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:37,840 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:37,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:37,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:37,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:37,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:37,840 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,844 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:37,844 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/A is initiating minor compaction (all files) 2024-11-13T18:25:37,844 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/A in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:37,844 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/51c4c78bf0ad432bb1a371aa6a1b161f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/c53bad85e2f046cfbff89897716c358c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/bacf4726792d43acbee1d761758b1ec9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/72c4e1289b644799b96e75136615d055] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=138.4 K 2024-11-13T18:25:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,844 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:37,844 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/51c4c78bf0ad432bb1a371aa6a1b161f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/c53bad85e2f046cfbff89897716c358c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/bacf4726792d43acbee1d761758b1ec9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/72c4e1289b644799b96e75136615d055] 2024-11-13T18:25:37,844 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:37,844 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/B is initiating minor compaction (all files) 2024-11-13T18:25:37,845 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/B in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:37,845 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c4c78bf0ad432bb1a371aa6a1b161f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731522331316 2024-11-13T18:25:37,845 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/657693235d4846f0976c4e152c2120d4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4c41580a4bfa4f4487870ae502005053, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e6128e7a18a44140b3cec4e787f14458, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/543abc6128704e2ba65a98794a8635ac] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=47.5 K 2024-11-13T18:25:37,845 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c53bad85e2f046cfbff89897716c358c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731522332143 2024-11-13T18:25:37,845 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 657693235d4846f0976c4e152c2120d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731522331316 2024-11-13T18:25:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,845 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting bacf4726792d43acbee1d761758b1ec9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731522333424 2024-11-13T18:25:37,845 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c41580a4bfa4f4487870ae502005053, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731522332157 2024-11-13T18:25:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,847 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6128e7a18a44140b3cec4e787f14458, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731522333424 2024-11-13T18:25:37,847 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c4e1289b644799b96e75136615d055, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731522335696 2024-11-13T18:25:37,847 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 543abc6128704e2ba65a98794a8635ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731522335696 2024-11-13T18:25:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,867 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,878 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#B#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:37,878 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/02ee9814ad1a4a029b964a36b1b29133 is 50, key is test_row_0/B:col10/1731522335696/Put/seqid=0 2024-11-13T18:25:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,884 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411135d0427b724014c7db2fc7f1a7a27a1ce_aa8bc3911d985a299e88f859af353050 store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,887 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411135d0427b724014c7db2fc7f1a7a27a1ce_aa8bc3911d985a299e88f859af353050, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:37,887 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135d0427b724014c7db2fc7f1a7a27a1ce_aa8bc3911d985a299e88f859af353050 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742213_1389 (size=4469) 2024-11-13T18:25:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,923 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#A#compaction#327 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,924 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/130b089406bc40c788ad7d40a3b61e90 is 175, key is test_row_0/A:col10/1731522335696/Put/seqid=0 2024-11-13T18:25:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742212_1388 (size=12493) 2024-11-13T18:25:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742214_1390 (size=31447) 2024-11-13T18:25:37,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:25:37,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,952 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:25:37,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:37,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:37,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:37,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:37,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:37,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130ece74561571410cbc6056269688f3d1_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_1/A:col10/1731522336404/Put/seqid=0 2024-11-13T18:25:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742215_1391 (size=9814) 2024-11-13T18:25:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,986 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130ece74561571410cbc6056269688f3d1_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130ece74561571410cbc6056269688f3d1_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/6ee8fbc5471b4ec79a41f989c317961c, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/6ee8fbc5471b4ec79a41f989c317961c is 175, key is test_row_1/A:col10/1731522336404/Put/seqid=0 2024-11-13T18:25:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742216_1392 (size=22461) 2024-11-13T18:25:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:25:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,334 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/02ee9814ad1a4a029b964a36b1b29133 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/02ee9814ad1a4a029b964a36b1b29133 2024-11-13T18:25:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,345 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa8bc3911d985a299e88f859af353050/B of aa8bc3911d985a299e88f859af353050 into 02ee9814ad1a4a029b964a36b1b29133(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,345 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:38,345 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/B, priority=12, startTime=1731522337840; duration=0sec 2024-11-13T18:25:38,345 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:38,345 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:B 2024-11-13T18:25:38,345 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,348 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:38,348 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/C is initiating minor compaction (all files) 2024-11-13T18:25:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,348 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/C in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,348 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/6a2fcc74356f43b3a21c4a6b0e5895a9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/baa29936c4784a88966bff6332608f19, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/93437ae6e2ca48c18073f379d7a0e366, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8bdb079cd04c43bdac5efb72003b8c12] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=47.5 K 2024-11-13T18:25:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,348 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a2fcc74356f43b3a21c4a6b0e5895a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731522331316 2024-11-13T18:25:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,349 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa29936c4784a88966bff6332608f19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731522332157 2024-11-13T18:25:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,349 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93437ae6e2ca48c18073f379d7a0e366, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731522333424 2024-11-13T18:25:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,349 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bdb079cd04c43bdac5efb72003b8c12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731522335696 2024-11-13T18:25:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,357 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/130b089406bc40c788ad7d40a3b61e90 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/130b089406bc40c788ad7d40a3b61e90 2024-11-13T18:25:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,363 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa8bc3911d985a299e88f859af353050/A of aa8bc3911d985a299e88f859af353050 into 130b089406bc40c788ad7d40a3b61e90(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:38,363 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:38,363 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/A, priority=12, startTime=1731522337840; duration=0sec 2024-11-13T18:25:38,363 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:38,363 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:A 2024-11-13T18:25:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,367 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#C#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,368 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/817db45c8d424a6eb66a199b81730290 is 50, key is test_row_0/C:col10/1731522335696/Put/seqid=0 2024-11-13T18:25:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742217_1393 (size=12493) 2024-11-13T18:25:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,392 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/6ee8fbc5471b4ec79a41f989c317961c 2024-11-13T18:25:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/48968cb074884398bcd8508c9dc2f8f0 is 50, key is test_row_1/B:col10/1731522336404/Put/seqid=0 2024-11-13T18:25:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742218_1394 (size=9757) 2024-11-13T18:25:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:25:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:38,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522398717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522398720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:38,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522398722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:38,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:38,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522398723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:38,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:38,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522398725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:38,815 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/817db45c8d424a6eb66a199b81730290 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/817db45c8d424a6eb66a199b81730290 2024-11-13T18:25:38,815 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/48968cb074884398bcd8508c9dc2f8f0 2024-11-13T18:25:38,836 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa8bc3911d985a299e88f859af353050/C of aa8bc3911d985a299e88f859af353050 into 817db45c8d424a6eb66a199b81730290(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:38,836 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:38,836 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/C, priority=12, startTime=1731522337840; duration=0sec 2024-11-13T18:25:38,836 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:38,836 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:C 2024-11-13T18:25:38,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/455bd6592ee74e818aa84a52cdedc1dc is 50, key is test_row_1/C:col10/1731522336404/Put/seqid=0 2024-11-13T18:25:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522398839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522398839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:38,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522398846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:38,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522398846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:38,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522398847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:38,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742219_1395 (size=9757) 2024-11-13T18:25:38,889 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/455bd6592ee74e818aa84a52cdedc1dc 2024-11-13T18:25:38,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/6ee8fbc5471b4ec79a41f989c317961c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/6ee8fbc5471b4ec79a41f989c317961c 2024-11-13T18:25:38,912 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/6ee8fbc5471b4ec79a41f989c317961c, entries=100, sequenceid=180, filesize=21.9 K 2024-11-13T18:25:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/48968cb074884398bcd8508c9dc2f8f0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/48968cb074884398bcd8508c9dc2f8f0 2024-11-13T18:25:38,924 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/48968cb074884398bcd8508c9dc2f8f0, entries=100, sequenceid=180, filesize=9.5 K 2024-11-13T18:25:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/455bd6592ee74e818aa84a52cdedc1dc as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/455bd6592ee74e818aa84a52cdedc1dc 2024-11-13T18:25:38,945 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/455bd6592ee74e818aa84a52cdedc1dc, entries=100, sequenceid=180, filesize=9.5 K 2024-11-13T18:25:38,946 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for aa8bc3911d985a299e88f859af353050 in 994ms, sequenceid=180, compaction requested=false 2024-11-13T18:25:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:38,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:38,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-13T18:25:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-13T18:25:38,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-13T18:25:38,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4710 sec 2024-11-13T18:25:38,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.4860 sec 2024-11-13T18:25:39,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522338715, max timestamp=1731522339073Num uniques 859; 2024-11-13T18:25:39,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:39,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-13T18:25:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:39,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:39,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:39,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:39,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137ff15c4e886a412dbd8e805fbf148088_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522339073/Put/seqid=0 2024-11-13T18:25:39,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522399131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522399132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522399134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742220_1396 (size=14794) 2024-11-13T18:25:39,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522399135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522399136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522399257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522399266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522399266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522399277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522399278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522399476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522399487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522399496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522399496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522399497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,563 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:39,569 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137ff15c4e886a412dbd8e805fbf148088_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137ff15c4e886a412dbd8e805fbf148088_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:39,570 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/10ef0f261b5240759ff71b779793717e, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:39,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/10ef0f261b5240759ff71b779793717e is 175, key is test_row_0/A:col10/1731522339073/Put/seqid=0 2024-11-13T18:25:39,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742221_1397 (size=39749) 2024-11-13T18:25:39,621 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/10ef0f261b5240759ff71b779793717e 2024-11-13T18:25:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:25:39,623 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:39,626 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-13T18:25:39,628 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:39,630 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:39,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:25:39,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/4632037eaf394e43aa49fe9832e8f904 is 50, key is test_row_0/B:col10/1731522339073/Put/seqid=0 2024-11-13T18:25:39,695 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:25:39,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742222_1398 (size=12151) 2024-11-13T18:25:39,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/4632037eaf394e43aa49fe9832e8f904 2024-11-13T18:25:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:25:39,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/633a090b6774457292a49e7d93e19c4d is 50, key is test_row_0/C:col10/1731522339073/Put/seqid=0 2024-11-13T18:25:39,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T18:25:39,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:39,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:39,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:39,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:39,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742223_1399 (size=12151) 2024-11-13T18:25:39,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/633a090b6774457292a49e7d93e19c4d 2024-11-13T18:25:39,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/10ef0f261b5240759ff71b779793717e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/10ef0f261b5240759ff71b779793717e 2024-11-13T18:25:39,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522399798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/10ef0f261b5240759ff71b779793717e, entries=200, sequenceid=204, filesize=38.8 K 2024-11-13T18:25:39,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/4632037eaf394e43aa49fe9832e8f904 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4632037eaf394e43aa49fe9832e8f904 2024-11-13T18:25:39,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4632037eaf394e43aa49fe9832e8f904, entries=150, sequenceid=204, filesize=11.9 K 2024-11-13T18:25:39,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522399817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/633a090b6774457292a49e7d93e19c4d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/633a090b6774457292a49e7d93e19c4d 2024-11-13T18:25:39,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522399817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522399829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522399834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:39,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/633a090b6774457292a49e7d93e19c4d, entries=150, sequenceid=204, filesize=11.9 K 2024-11-13T18:25:39,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for aa8bc3911d985a299e88f859af353050 in 768ms, sequenceid=204, compaction requested=true 2024-11-13T18:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:39,851 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:39,851 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:39,869 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:39,869 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/A is initiating minor compaction (all files) 2024-11-13T18:25:39,870 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/A in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:39,870 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/130b089406bc40c788ad7d40a3b61e90, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/6ee8fbc5471b4ec79a41f989c317961c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/10ef0f261b5240759ff71b779793717e] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=91.5 K 2024-11-13T18:25:39,870 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:39,870 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/130b089406bc40c788ad7d40a3b61e90, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/6ee8fbc5471b4ec79a41f989c317961c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/10ef0f261b5240759ff71b779793717e] 2024-11-13T18:25:39,876 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 130b089406bc40c788ad7d40a3b61e90, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731522335696 2024-11-13T18:25:39,878 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:39,878 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/B is initiating minor compaction (all files) 2024-11-13T18:25:39,878 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/B in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:39,879 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/02ee9814ad1a4a029b964a36b1b29133, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/48968cb074884398bcd8508c9dc2f8f0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4632037eaf394e43aa49fe9832e8f904] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=33.6 K 2024-11-13T18:25:39,883 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ee8fbc5471b4ec79a41f989c317961c, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731522336404 2024-11-13T18:25:39,885 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10ef0f261b5240759ff71b779793717e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731522338715 2024-11-13T18:25:39,887 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ee9814ad1a4a029b964a36b1b29133, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731522335696 2024-11-13T18:25:39,888 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 48968cb074884398bcd8508c9dc2f8f0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731522336404 2024-11-13T18:25:39,889 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 4632037eaf394e43aa49fe9832e8f904, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731522338721 2024-11-13T18:25:39,905 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:39,925 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#B#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:39,925 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e92111bcda4642ddb261798181441515 is 50, key is test_row_0/B:col10/1731522339073/Put/seqid=0 2024-11-13T18:25:39,933 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113cddade61fdb44b708f13bf34ab0fa746_aa8bc3911d985a299e88f859af353050 store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:39,935 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113cddade61fdb44b708f13bf34ab0fa746_aa8bc3911d985a299e88f859af353050, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:39,936 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cddade61fdb44b708f13bf34ab0fa746_aa8bc3911d985a299e88f859af353050 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:39,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T18:25:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:39,941 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:25:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:39,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:39,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:25:39,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742224_1400 (size=12595) 2024-11-13T18:25:40,014 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e92111bcda4642ddb261798181441515 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e92111bcda4642ddb261798181441515 2024-11-13T18:25:40,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742225_1401 (size=4469) 2024-11-13T18:25:40,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132549cbc2e0044b2ca4974f7d68d62869_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522339132/Put/seqid=0 2024-11-13T18:25:40,021 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#A#compaction#336 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:40,022 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/904f621d00e2446283a92491fd4e4e93 is 175, key is test_row_0/A:col10/1731522339073/Put/seqid=0 2024-11-13T18:25:40,027 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/B of aa8bc3911d985a299e88f859af353050 into e92111bcda4642ddb261798181441515(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:40,027 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:40,027 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/B, priority=13, startTime=1731522339851; duration=0sec 2024-11-13T18:25:40,028 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:40,028 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:B 2024-11-13T18:25:40,028 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:40,029 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:40,029 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/C is initiating minor compaction (all files) 2024-11-13T18:25:40,030 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/C in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:40,030 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/817db45c8d424a6eb66a199b81730290, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/455bd6592ee74e818aa84a52cdedc1dc, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/633a090b6774457292a49e7d93e19c4d] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=33.6 K 2024-11-13T18:25:40,033 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 817db45c8d424a6eb66a199b81730290, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731522335696 2024-11-13T18:25:40,038 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 455bd6592ee74e818aa84a52cdedc1dc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731522336404 2024-11-13T18:25:40,041 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 633a090b6774457292a49e7d93e19c4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731522338721 2024-11-13T18:25:40,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742226_1402 (size=12304) 2024-11-13T18:25:40,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:40,076 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132549cbc2e0044b2ca4974f7d68d62869_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132549cbc2e0044b2ca4974f7d68d62869_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:40,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/d82a23d9df854a95bd0750479cb7c605, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:40,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/d82a23d9df854a95bd0750479cb7c605 is 175, key is test_row_0/A:col10/1731522339132/Put/seqid=0 2024-11-13T18:25:40,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742227_1403 (size=31549) 2024-11-13T18:25:40,106 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:40,106 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/cb68db0b821d4885a1e19320dd34f18c is 50, key is test_row_0/C:col10/1731522339073/Put/seqid=0 2024-11-13T18:25:40,119 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/904f621d00e2446283a92491fd4e4e93 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/904f621d00e2446283a92491fd4e4e93 2024-11-13T18:25:40,131 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/A of aa8bc3911d985a299e88f859af353050 into 904f621d00e2446283a92491fd4e4e93(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:40,131 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:40,131 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/A, priority=13, startTime=1731522339851; duration=0sec 2024-11-13T18:25:40,131 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:40,131 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:A 2024-11-13T18:25:40,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742228_1404 (size=31105) 2024-11-13T18:25:40,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742229_1405 (size=12595) 2024-11-13T18:25:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:25:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:40,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:40,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522400418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522400421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522400422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522400423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522400444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,560 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/d82a23d9df854a95bd0750479cb7c605 2024-11-13T18:25:40,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522400557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522400557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522400557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522400557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,575 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/cb68db0b821d4885a1e19320dd34f18c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/cb68db0b821d4885a1e19320dd34f18c 2024-11-13T18:25:40,585 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/C of aa8bc3911d985a299e88f859af353050 into cb68db0b821d4885a1e19320dd34f18c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:40,585 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:40,585 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/C, priority=13, startTime=1731522339851; duration=0sec 2024-11-13T18:25:40,585 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:40,585 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:C 2024-11-13T18:25:40,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522400576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/0ff551856280448485b63959f50f025b is 50, key is test_row_0/B:col10/1731522339132/Put/seqid=0 2024-11-13T18:25:40,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742230_1406 (size=12151) 2024-11-13T18:25:40,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:25:40,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522400776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522400777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522400777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522400777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:40,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522400796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,033 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/0ff551856280448485b63959f50f025b 2024-11-13T18:25:41,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/411f1c0cf7fb424a8697b25c47f04a2e is 50, key is test_row_0/C:col10/1731522339132/Put/seqid=0 2024-11-13T18:25:41,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522401098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522401107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522401107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742231_1407 (size=12151) 2024-11-13T18:25:41,130 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/411f1c0cf7fb424a8697b25c47f04a2e 2024-11-13T18:25:41,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522401107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522401121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/d82a23d9df854a95bd0750479cb7c605 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/d82a23d9df854a95bd0750479cb7c605 2024-11-13T18:25:41,156 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/d82a23d9df854a95bd0750479cb7c605, entries=150, sequenceid=220, filesize=30.4 K 2024-11-13T18:25:41,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/0ff551856280448485b63959f50f025b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/0ff551856280448485b63959f50f025b 2024-11-13T18:25:41,184 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/0ff551856280448485b63959f50f025b, entries=150, sequenceid=220, filesize=11.9 K 2024-11-13T18:25:41,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/411f1c0cf7fb424a8697b25c47f04a2e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/411f1c0cf7fb424a8697b25c47f04a2e 2024-11-13T18:25:41,193 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/411f1c0cf7fb424a8697b25c47f04a2e, entries=150, sequenceid=220, filesize=11.9 K 2024-11-13T18:25:41,195 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for aa8bc3911d985a299e88f859af353050 in 1254ms, sequenceid=220, compaction requested=false 2024-11-13T18:25:41,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:41,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:41,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-13T18:25:41,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-13T18:25:41,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-13T18:25:41,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-11-13T18:25:41,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.5760 sec 2024-11-13T18:25:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:41,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:25:41,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:41,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:41,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:41,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:41,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:41,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:41,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132c2b99cd80334fb2a88e81a4e1d3e11c_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522340414/Put/seqid=0 2024-11-13T18:25:41,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742232_1408 (size=14794) 2024-11-13T18:25:41,712 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:41,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522401687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522401674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132c2b99cd80334fb2a88e81a4e1d3e11c_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c2b99cd80334fb2a88e81a4e1d3e11c_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:41,717 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/5e3ed0713a0d4855a7eac1dd72ebcd4f, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:41,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/5e3ed0713a0d4855a7eac1dd72ebcd4f is 175, key is test_row_0/A:col10/1731522340414/Put/seqid=0 2024-11-13T18:25:41,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522401703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522401714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522401715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742233_1409 (size=39749) 2024-11-13T18:25:41,775 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/5e3ed0713a0d4855a7eac1dd72ebcd4f 2024-11-13T18:25:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:25:41,786 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:41,787 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-13T18:25:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:25:41,794 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:41,796 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:41,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:41,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/5e80b399dada421e8237cdf838a81552 is 50, key is test_row_0/B:col10/1731522340414/Put/seqid=0 2024-11-13T18:25:41,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522401818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522401825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522401836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522401848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522401851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:41,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742234_1410 (size=12151) 2024-11-13T18:25:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:25:41,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:41,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:41,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522402045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522402059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522402059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522402081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522402081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:25:42,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:42,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:42,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/5e80b399dada421e8237cdf838a81552 2024-11-13T18:25:42,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/8b8e6c9e929041838c41c5ca8371225e is 50, key is test_row_0/C:col10/1731522340414/Put/seqid=0 2024-11-13T18:25:42,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742235_1411 (size=12151) 2024-11-13T18:25:42,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/8b8e6c9e929041838c41c5ca8371225e 2024-11-13T18:25:42,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/5e3ed0713a0d4855a7eac1dd72ebcd4f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5e3ed0713a0d4855a7eac1dd72ebcd4f 2024-11-13T18:25:42,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5e3ed0713a0d4855a7eac1dd72ebcd4f, entries=200, sequenceid=240, filesize=38.8 K 2024-11-13T18:25:42,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/5e80b399dada421e8237cdf838a81552 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5e80b399dada421e8237cdf838a81552 2024-11-13T18:25:42,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5e80b399dada421e8237cdf838a81552, entries=150, sequenceid=240, filesize=11.9 K 2024-11-13T18:25:42,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/8b8e6c9e929041838c41c5ca8371225e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8b8e6c9e929041838c41c5ca8371225e 2024-11-13T18:25:42,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8b8e6c9e929041838c41c5ca8371225e, entries=150, sequenceid=240, filesize=11.9 K 2024-11-13T18:25:42,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for aa8bc3911d985a299e88f859af353050 in 772ms, sequenceid=240, compaction requested=true 2024-11-13T18:25:42,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:42,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:42,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:42,399 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:42,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:42,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:42,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:42,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:42,399 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:42,402 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:42,402 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/A is initiating minor compaction (all files) 2024-11-13T18:25:42,403 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/A in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,403 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/904f621d00e2446283a92491fd4e4e93, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/d82a23d9df854a95bd0750479cb7c605, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5e3ed0713a0d4855a7eac1dd72ebcd4f] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=100.0 K 2024-11-13T18:25:42,403 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,403 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/904f621d00e2446283a92491fd4e4e93, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/d82a23d9df854a95bd0750479cb7c605, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5e3ed0713a0d4855a7eac1dd72ebcd4f] 2024-11-13T18:25:42,403 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:42,403 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 904f621d00e2446283a92491fd4e4e93, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731522338721 2024-11-13T18:25:42,403 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/B is initiating minor compaction (all files) 2024-11-13T18:25:42,404 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/B in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,404 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e92111bcda4642ddb261798181441515, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/0ff551856280448485b63959f50f025b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5e80b399dada421e8237cdf838a81552] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=36.0 K 2024-11-13T18:25:42,404 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting d82a23d9df854a95bd0750479cb7c605, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731522339132 2024-11-13T18:25:42,404 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e3ed0713a0d4855a7eac1dd72ebcd4f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731522340414 2024-11-13T18:25:42,404 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e92111bcda4642ddb261798181441515, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731522338721 2024-11-13T18:25:42,405 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff551856280448485b63959f50f025b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731522339132 2024-11-13T18:25:42,405 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e80b399dada421e8237cdf838a81552, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731522340414 2024-11-13T18:25:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:42,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:25:42,415 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:42,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:42,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:42,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:42,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:42,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:42,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:25:42,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,424 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#B#compaction#346 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:42,424 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/f6e64bb64b87424484f620704cd836dc is 50, key is test_row_0/B:col10/1731522340414/Put/seqid=0 2024-11-13T18:25:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,426 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411132aac8bca5c884a37b81e512945bf863c_aa8bc3911d985a299e88f859af353050 store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:42,428 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411132aac8bca5c884a37b81e512945bf863c_aa8bc3911d985a299e88f859af353050, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:42,428 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132aac8bca5c884a37b81e512945bf863c_aa8bc3911d985a299e88f859af353050 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:42,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742236_1412 (size=12697) 2024-11-13T18:25:42,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c4e413c9ae884aa8bb917f3bc3794881_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522342406/Put/seqid=0 2024-11-13T18:25:42,475 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/f6e64bb64b87424484f620704cd836dc as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f6e64bb64b87424484f620704cd836dc 2024-11-13T18:25:42,490 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/B of aa8bc3911d985a299e88f859af353050 into f6e64bb64b87424484f620704cd836dc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:42,490 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:42,490 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/B, priority=13, startTime=1731522342399; duration=0sec 2024-11-13T18:25:42,491 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:42,491 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:B 2024-11-13T18:25:42,493 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:42,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,496 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522402459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,497 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/C is initiating minor compaction (all files) 2024-11-13T18:25:42,497 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/C in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,497 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/cb68db0b821d4885a1e19320dd34f18c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/411f1c0cf7fb424a8697b25c47f04a2e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8b8e6c9e929041838c41c5ca8371225e] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=36.0 K 2024-11-13T18:25:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522402469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,498 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting cb68db0b821d4885a1e19320dd34f18c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731522338721 2024-11-13T18:25:42,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742237_1413 (size=4469) 2024-11-13T18:25:42,499 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 411f1c0cf7fb424a8697b25c47f04a2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731522339132 2024-11-13T18:25:42,500 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b8e6c9e929041838c41c5ca8371225e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731522340414 2024-11-13T18:25:42,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522402494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522402496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522402496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,570 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#C#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:42,571 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/01dd6ecd58fa40fe84febdef0f73d02c is 50, key is test_row_0/C:col10/1731522340414/Put/seqid=0 2024-11-13T18:25:42,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742238_1414 (size=12354) 2024-11-13T18:25:42,588 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:42,595 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c4e413c9ae884aa8bb917f3bc3794881_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c4e413c9ae884aa8bb917f3bc3794881_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:42,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/ae5eec4e71b34dd88f9b79f717e21bc6, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:42,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/ae5eec4e71b34dd88f9b79f717e21bc6 is 175, key is test_row_0/A:col10/1731522342406/Put/seqid=0 2024-11-13T18:25:42,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522402606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522402606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742239_1415 (size=12697) 2024-11-13T18:25:42,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522402636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742240_1416 (size=31155) 2024-11-13T18:25:42,654 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/ae5eec4e71b34dd88f9b79f717e21bc6 2024-11-13T18:25:42,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522402636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522402646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/5375e115574042c087fe1b36a4ee2193 is 50, key is test_row_0/B:col10/1731522342406/Put/seqid=0 2024-11-13T18:25:42,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742241_1417 (size=12201) 2024-11-13T18:25:42,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/5375e115574042c087fe1b36a4ee2193 2024-11-13T18:25:42,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/b671e7ba8b284fa593bb062daccebf9a is 50, key is test_row_0/C:col10/1731522342406/Put/seqid=0 2024-11-13T18:25:42,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742242_1418 (size=12201) 2024-11-13T18:25:42,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522402839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522402839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522402867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522402867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522402868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:42,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:42,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:42,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:42,904 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#A#compaction#345 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:42,905 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/8cedf8866a6242d0a0a4be3dc40422ac is 175, key is test_row_0/A:col10/1731522340414/Put/seqid=0 2024-11-13T18:25:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:25:42,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742243_1419 (size=31651) 2024-11-13T18:25:43,042 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/01dd6ecd58fa40fe84febdef0f73d02c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01dd6ecd58fa40fe84febdef0f73d02c 2024-11-13T18:25:43,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,054 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/C of aa8bc3911d985a299e88f859af353050 into 01dd6ecd58fa40fe84febdef0f73d02c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:43,054 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:43,054 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/C, priority=13, startTime=1731522342399; duration=0sec 2024-11-13T18:25:43,054 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:43,055 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:C 2024-11-13T18:25:43,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522403166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522403166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/b671e7ba8b284fa593bb062daccebf9a 2024-11-13T18:25:43,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/ae5eec4e71b34dd88f9b79f717e21bc6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/ae5eec4e71b34dd88f9b79f717e21bc6 2024-11-13T18:25:43,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/ae5eec4e71b34dd88f9b79f717e21bc6, entries=150, sequenceid=258, filesize=30.4 K 2024-11-13T18:25:43,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/5375e115574042c087fe1b36a4ee2193 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5375e115574042c087fe1b36a4ee2193 2024-11-13T18:25:43,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522403198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5375e115574042c087fe1b36a4ee2193, entries=150, sequenceid=258, filesize=11.9 K 2024-11-13T18:25:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/b671e7ba8b284fa593bb062daccebf9a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/b671e7ba8b284fa593bb062daccebf9a 2024-11-13T18:25:43,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/b671e7ba8b284fa593bb062daccebf9a, entries=150, sequenceid=258, filesize=11.9 K 2024-11-13T18:25:43,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for aa8bc3911d985a299e88f859af353050 in 819ms, sequenceid=258, compaction requested=false 2024-11-13T18:25:43,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:43,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:25:43,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:43,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:43,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:43,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:43,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:43,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:43,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134448e71455784134bab21cef854cb066_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522342454/Put/seqid=0 2024-11-13T18:25:43,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742244_1420 (size=14994) 2024-11-13T18:25:43,320 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:43,328 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134448e71455784134bab21cef854cb066_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134448e71455784134bab21cef854cb066_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:43,329 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/7c326117762a4b27bb11897b352fd425, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:43,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/7c326117762a4b27bb11897b352fd425 is 175, key is test_row_0/A:col10/1731522342454/Put/seqid=0 2024-11-13T18:25:43,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742245_1421 (size=39949) 2024-11-13T18:25:43,360 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/7c326117762a4b27bb11897b352fd425 2024-11-13T18:25:43,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:43,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:43,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522403357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,384 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/8cedf8866a6242d0a0a4be3dc40422ac as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/8cedf8866a6242d0a0a4be3dc40422ac 2024-11-13T18:25:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522403363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,394 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/A of aa8bc3911d985a299e88f859af353050 into 8cedf8866a6242d0a0a4be3dc40422ac(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:43,394 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:43,394 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/A, priority=13, startTime=1731522342399; duration=0sec 2024-11-13T18:25:43,394 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:43,394 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:A 2024-11-13T18:25:43,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/c60757e9032149ed859769a4f225e2d1 is 50, key is test_row_0/B:col10/1731522342454/Put/seqid=0 2024-11-13T18:25:43,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742246_1422 (size=12301) 2024-11-13T18:25:43,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/c60757e9032149ed859769a4f225e2d1 2024-11-13T18:25:43,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/52c18f2f270e4299bfa010e93dd0c1cb is 50, key is test_row_0/C:col10/1731522342454/Put/seqid=0 2024-11-13T18:25:43,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742247_1423 (size=12301) 2024-11-13T18:25:43,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/52c18f2f270e4299bfa010e93dd0c1cb 2024-11-13T18:25:43,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/7c326117762a4b27bb11897b352fd425 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/7c326117762a4b27bb11897b352fd425 2024-11-13T18:25:43,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/7c326117762a4b27bb11897b352fd425, entries=200, sequenceid=276, filesize=39.0 K 2024-11-13T18:25:43,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/c60757e9032149ed859769a4f225e2d1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/c60757e9032149ed859769a4f225e2d1 2024-11-13T18:25:43,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:43,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522403496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/c60757e9032149ed859769a4f225e2d1, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T18:25:43,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/52c18f2f270e4299bfa010e93dd0c1cb as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/52c18f2f270e4299bfa010e93dd0c1cb 2024-11-13T18:25:43,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522403496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/52c18f2f270e4299bfa010e93dd0c1cb, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T18:25:43,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for aa8bc3911d985a299e88f859af353050 in 305ms, sequenceid=276, compaction requested=true 2024-11-13T18:25:43,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:43,537 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:43,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:43,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:43,538 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:43,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:43,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:43,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:43,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:43,538 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:43,538 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/A is initiating minor compaction (all files) 2024-11-13T18:25:43,539 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/A in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,539 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/8cedf8866a6242d0a0a4be3dc40422ac, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/ae5eec4e71b34dd88f9b79f717e21bc6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/7c326117762a4b27bb11897b352fd425] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=100.3 K 2024-11-13T18:25:43,539 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,539 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/8cedf8866a6242d0a0a4be3dc40422ac, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/ae5eec4e71b34dd88f9b79f717e21bc6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/7c326117762a4b27bb11897b352fd425] 2024-11-13T18:25:43,539 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:43,539 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/B is initiating minor compaction (all files) 2024-11-13T18:25:43,539 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cedf8866a6242d0a0a4be3dc40422ac, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731522340414 2024-11-13T18:25:43,539 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/B in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,539 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f6e64bb64b87424484f620704cd836dc, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5375e115574042c087fe1b36a4ee2193, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/c60757e9032149ed859769a4f225e2d1] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=36.3 K 2024-11-13T18:25:43,540 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting ae5eec4e71b34dd88f9b79f717e21bc6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731522341677 2024-11-13T18:25:43,540 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6e64bb64b87424484f620704cd836dc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731522340414 2024-11-13T18:25:43,540 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c326117762a4b27bb11897b352fd425, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731522342454 2024-11-13T18:25:43,541 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5375e115574042c087fe1b36a4ee2193, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731522341677 2024-11-13T18:25:43,541 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60757e9032149ed859769a4f225e2d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731522342454 2024-11-13T18:25:43,555 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:43,565 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#B#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:43,566 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/cc930f003b41423991fe053f51e3ac27 is 50, key is test_row_0/B:col10/1731522342454/Put/seqid=0 2024-11-13T18:25:43,578 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113107a60e0a04c4b238c2814d213ba1dc5_aa8bc3911d985a299e88f859af353050 store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:43,580 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113107a60e0a04c4b238c2814d213ba1dc5_aa8bc3911d985a299e88f859af353050, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:43,580 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113107a60e0a04c4b238c2814d213ba1dc5_aa8bc3911d985a299e88f859af353050 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:43,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742248_1424 (size=12949) 2024-11-13T18:25:43,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742249_1425 (size=4469) 2024-11-13T18:25:43,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:25:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:43,685 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:25:43,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:43,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:43,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:43,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:43,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:43,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:43,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137a531d8792a34de794bba377f1055679_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522343355/Put/seqid=0 2024-11-13T18:25:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:43,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742250_1426 (size=12454) 2024-11-13T18:25:43,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. as already flushing 2024-11-13T18:25:43,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522403781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522403783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522403784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522403811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522403814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522403920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:25:43,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522403925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522403926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522403955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522403959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:43,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,035 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/cc930f003b41423991fe053f51e3ac27 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/cc930f003b41423991fe053f51e3ac27 2024-11-13T18:25:44,041 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/B of aa8bc3911d985a299e88f859af353050 into cc930f003b41423991fe053f51e3ac27(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:44,041 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:44,041 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/B, priority=13, startTime=1731522343537; duration=0sec 2024-11-13T18:25:44,041 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:44,041 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:B 2024-11-13T18:25:44,041 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:44,042 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:44,042 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/C is initiating minor compaction (all files) 2024-11-13T18:25:44,042 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/C in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:44,042 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01dd6ecd58fa40fe84febdef0f73d02c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/b671e7ba8b284fa593bb062daccebf9a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/52c18f2f270e4299bfa010e93dd0c1cb] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=36.3 K 2024-11-13T18:25:44,043 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01dd6ecd58fa40fe84febdef0f73d02c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731522340414 2024-11-13T18:25:44,043 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b671e7ba8b284fa593bb062daccebf9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731522341677 2024-11-13T18:25:44,044 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52c18f2f270e4299bfa010e93dd0c1cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731522342454 2024-11-13T18:25:44,054 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#A#compaction#354 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:44,055 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/4c1ed9661bc343f18877edc5e0372ba0 is 175, key is test_row_0/A:col10/1731522342454/Put/seqid=0 2024-11-13T18:25:44,089 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#C#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:44,089 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/487367d1c0434888828e31e3c0b68ee2 is 50, key is test_row_0/C:col10/1731522342454/Put/seqid=0 2024-11-13T18:25:44,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742251_1427 (size=31903) 2024-11-13T18:25:44,106 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/4c1ed9661bc343f18877edc5e0372ba0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/4c1ed9661bc343f18877edc5e0372ba0 2024-11-13T18:25:44,115 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/A of aa8bc3911d985a299e88f859af353050 into 4c1ed9661bc343f18877edc5e0372ba0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:44,116 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:44,116 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/A, priority=13, startTime=1731522343537; duration=0sec 2024-11-13T18:25:44,117 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:44,117 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:A 2024-11-13T18:25:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:44,132 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137a531d8792a34de794bba377f1055679_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137a531d8792a34de794bba377f1055679_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:44,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/19180115f55c4a9297dba10e84c9b988, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/19180115f55c4a9297dba10e84c9b988 is 175, key is test_row_0/A:col10/1731522343355/Put/seqid=0 2024-11-13T18:25:44,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742252_1428 (size=12949) 2024-11-13T18:25:44,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522404148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522404148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522404149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742253_1429 (size=31255) 2024-11-13T18:25:44,178 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/487367d1c0434888828e31e3c0b68ee2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/487367d1c0434888828e31e3c0b68ee2 2024-11-13T18:25:44,180 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/19180115f55c4a9297dba10e84c9b988 2024-11-13T18:25:44,184 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/C of aa8bc3911d985a299e88f859af353050 into 487367d1c0434888828e31e3c0b68ee2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:44,184 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:44,184 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/C, priority=13, startTime=1731522343538; duration=0sec 2024-11-13T18:25:44,185 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:44,185 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:C 2024-11-13T18:25:44,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522404175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522404175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/bb76a9606bcf4e5d863fbc4e3c418941 is 50, key is test_row_0/B:col10/1731522343355/Put/seqid=0 2024-11-13T18:25:44,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742254_1430 (size=12301) 2024-11-13T18:25:44,233 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/bb76a9606bcf4e5d863fbc4e3c418941 2024-11-13T18:25:44,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/4da9092c5f6245f0a81b840ab3b154e2 is 50, key is test_row_0/C:col10/1731522343355/Put/seqid=0 2024-11-13T18:25:44,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742255_1431 (size=12301) 2024-11-13T18:25:44,274 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/4da9092c5f6245f0a81b840ab3b154e2 2024-11-13T18:25:44,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/19180115f55c4a9297dba10e84c9b988 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/19180115f55c4a9297dba10e84c9b988 2024-11-13T18:25:44,302 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/19180115f55c4a9297dba10e84c9b988, entries=150, sequenceid=293, filesize=30.5 K 2024-11-13T18:25:44,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/bb76a9606bcf4e5d863fbc4e3c418941 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/bb76a9606bcf4e5d863fbc4e3c418941 2024-11-13T18:25:44,323 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/bb76a9606bcf4e5d863fbc4e3c418941, entries=150, sequenceid=293, filesize=12.0 K 2024-11-13T18:25:44,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/4da9092c5f6245f0a81b840ab3b154e2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/4da9092c5f6245f0a81b840ab3b154e2 2024-11-13T18:25:44,333 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/4da9092c5f6245f0a81b840ab3b154e2, entries=150, sequenceid=293, filesize=12.0 K 2024-11-13T18:25:44,336 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for aa8bc3911d985a299e88f859af353050 in 652ms, sequenceid=293, compaction requested=false 2024-11-13T18:25:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-13T18:25:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-13T18:25:44,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-13T18:25:44,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5470 sec 2024-11-13T18:25:44,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.5610 sec 2024-11-13T18:25:44,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:25:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:44,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133b943de2606447bdabb074155d74e2ad_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522343754/Put/seqid=0 2024-11-13T18:25:44,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742256_1432 (size=14994) 2024-11-13T18:25:44,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522404524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,547 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:44,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522404534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522404538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,561 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133b943de2606447bdabb074155d74e2ad_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133b943de2606447bdabb074155d74e2ad_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522404545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522404546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,566 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/65343a03422d41d2aeb0d66b6c64f74f, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:44,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/65343a03422d41d2aeb0d66b6c64f74f is 175, key is test_row_0/A:col10/1731522343754/Put/seqid=0 2024-11-13T18:25:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742257_1433 (size=39949) 2024-11-13T18:25:44,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522404662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522404665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522404671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522404676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522404676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50510 deadline: 1731522404891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50508 deadline: 1731522404894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50588 deadline: 1731522404899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50574 deadline: 1731522404899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:44,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50546 deadline: 1731522404899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:44,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:44,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:25:45,010 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/65343a03422d41d2aeb0d66b6c64f74f 2024-11-13T18:25:45,042 INFO [Thread-1595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1595. 2024-11-13T18:25:45,042 DEBUG [Thread-1595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:45,042 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,042 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,042 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-11-13T18:25:45,042 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:45,042 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,043 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e50d9dd4e86645478ca80060ce59a3db is 50, key is test_row_0/B:col10/1731522343754/Put/seqid=0 2024-11-13T18:25:45,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,048 INFO [Thread-1597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1597. 2024-11-13T18:25:45,048 DEBUG [Thread-1597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:45,048 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,048 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,060 INFO [Thread-1593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1593. 2024-11-13T18:25:45,061 DEBUG [Thread-1593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:45,061 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742258_1434 (size=12301) 2024-11-13T18:25:45,061 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,069 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-11-13T18:25:45,070 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:45,070 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,070 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e50d9dd4e86645478ca80060ce59a3db 2024-11-13T18:25:45,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/62b4cac5f09041f493094381cf36612a is 50, key is test_row_0/C:col10/1731522343754/Put/seqid=0 2024-11-13T18:25:45,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742259_1435 (size=12301) 2024-11-13T18:25:45,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/62b4cac5f09041f493094381cf36612a 2024-11-13T18:25:45,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/65343a03422d41d2aeb0d66b6c64f74f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/65343a03422d41d2aeb0d66b6c64f74f 2024-11-13T18:25:45,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/65343a03422d41d2aeb0d66b6c64f74f, entries=200, sequenceid=313, filesize=39.0 K 2024-11-13T18:25:45,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/e50d9dd4e86645478ca80060ce59a3db as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e50d9dd4e86645478ca80060ce59a3db 2024-11-13T18:25:45,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e50d9dd4e86645478ca80060ce59a3db, entries=150, sequenceid=313, filesize=12.0 K 2024-11-13T18:25:45,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/62b4cac5f09041f493094381cf36612a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/62b4cac5f09041f493094381cf36612a 2024-11-13T18:25:45,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/62b4cac5f09041f493094381cf36612a, entries=150, sequenceid=313, filesize=12.0 K 2024-11-13T18:25:45,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for aa8bc3911d985a299e88f859af353050 in 697ms, sequenceid=313, compaction requested=true 2024-11-13T18:25:45,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:45,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:45,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:45,174 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:45,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:45,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:45,174 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:45,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa8bc3911d985a299e88f859af353050:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:45,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:45,181 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:45,181 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/A is initiating minor compaction (all files) 2024-11-13T18:25:45,181 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:45,181 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/A in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:45,181 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/B is initiating minor compaction (all files) 2024-11-13T18:25:45,181 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/4c1ed9661bc343f18877edc5e0372ba0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/19180115f55c4a9297dba10e84c9b988, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/65343a03422d41d2aeb0d66b6c64f74f] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=100.7 K 2024-11-13T18:25:45,181 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/B in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:45,181 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:45,181 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/cc930f003b41423991fe053f51e3ac27, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/bb76a9606bcf4e5d863fbc4e3c418941, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e50d9dd4e86645478ca80060ce59a3db] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=36.7 K 2024-11-13T18:25:45,181 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/4c1ed9661bc343f18877edc5e0372ba0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/19180115f55c4a9297dba10e84c9b988, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/65343a03422d41d2aeb0d66b6c64f74f] 2024-11-13T18:25:45,183 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting cc930f003b41423991fe053f51e3ac27, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731522342454 2024-11-13T18:25:45,183 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c1ed9661bc343f18877edc5e0372ba0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731522342454 2024-11-13T18:25:45,185 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting bb76a9606bcf4e5d863fbc4e3c418941, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731522343271 2024-11-13T18:25:45,185 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19180115f55c4a9297dba10e84c9b988, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731522343271 2024-11-13T18:25:45,187 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e50d9dd4e86645478ca80060ce59a3db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731522343754 2024-11-13T18:25:45,187 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65343a03422d41d2aeb0d66b6c64f74f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731522343754 2024-11-13T18:25:45,208 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:45,209 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/87621b9911ba45fa802f8aeecef21611 is 50, key is test_row_0/B:col10/1731522343754/Put/seqid=0 2024-11-13T18:25:45,215 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:45,223 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-13T18:25:45,224 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:45,224 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,224 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,224 INFO [Thread-1584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1584. 2024-11-13T18:25:45,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,224 DEBUG [Thread-1584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:45,224 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,224 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,225 INFO [Thread-1582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1582. 2024-11-13T18:25:45,226 DEBUG [Thread-1582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:45,226 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,227 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa8bc3911d985a299e88f859af353050 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:25:45,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=A 2024-11-13T18:25:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=B 2024-11-13T18:25:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa8bc3911d985a299e88f859af353050, store=C 2024-11-13T18:25:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:45,234 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-11-13T18:25:45,234 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:45,234 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,234 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,237 INFO [Thread-1586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1586. 2024-11-13T18:25:45,237 DEBUG [Thread-1586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:25:45,237 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,237 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742260_1436 (size=13051) 2024-11-13T18:25:45,258 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113d329e8c36aed48639289427176f7afe0_aa8bc3911d985a299e88f859af353050 store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:45,273 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113d329e8c36aed48639289427176f7afe0_aa8bc3911d985a299e88f859af353050, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:45,273 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d329e8c36aed48639289427176f7afe0_aa8bc3911d985a299e88f859af353050 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:45,280 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/87621b9911ba45fa802f8aeecef21611 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/87621b9911ba45fa802f8aeecef21611 2024-11-13T18:25:45,298 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/B of aa8bc3911d985a299e88f859af353050 into 87621b9911ba45fa802f8aeecef21611(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:45,298 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:45,298 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/B, priority=13, startTime=1731522345174; duration=0sec 2024-11-13T18:25:45,299 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:45,299 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:B 2024-11-13T18:25:45,299 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:45,300 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:45,300 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): aa8bc3911d985a299e88f859af353050/C is initiating minor compaction (all files) 2024-11-13T18:25:45,300 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa8bc3911d985a299e88f859af353050/C in TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:45,300 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/487367d1c0434888828e31e3c0b68ee2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/4da9092c5f6245f0a81b840ab3b154e2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/62b4cac5f09041f493094381cf36612a] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp, totalSize=36.7 K 2024-11-13T18:25:45,301 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 487367d1c0434888828e31e3c0b68ee2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731522342454 2024-11-13T18:25:45,301 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 4da9092c5f6245f0a81b840ab3b154e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731522343271 2024-11-13T18:25:45,301 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 62b4cac5f09041f493094381cf36612a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731522343754 2024-11-13T18:25:45,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111389cad3aa41ff4ef9bf6170bc1eb6d954_aa8bc3911d985a299e88f859af353050 is 50, key is test_row_0/A:col10/1731522344530/Put/seqid=0 2024-11-13T18:25:45,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742261_1437 (size=4469) 2024-11-13T18:25:45,340 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#A#compaction#364 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:45,341 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/c4184599f81848dd95f2b3e0c4f67c1c is 175, key is test_row_0/A:col10/1731522343754/Put/seqid=0 2024-11-13T18:25:45,359 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa8bc3911d985a299e88f859af353050#C#compaction#366 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:45,360 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/2951a1be82d24fa0841076a8e4c88f7a is 50, key is test_row_0/C:col10/1731522343754/Put/seqid=0 2024-11-13T18:25:45,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742262_1438 (size=12454) 2024-11-13T18:25:45,451 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:45,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742264_1440 (size=13051) 2024-11-13T18:25:45,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742263_1439 (size=32005) 2024-11-13T18:25:45,465 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111389cad3aa41ff4ef9bf6170bc1eb6d954_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111389cad3aa41ff4ef9bf6170bc1eb6d954_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:45,469 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/c4184599f81848dd95f2b3e0c4f67c1c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/c4184599f81848dd95f2b3e0c4f67c1c 2024-11-13T18:25:45,469 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/5caa9813a2ce4359a898e16d6b9f52d8, store: [table=TestAcidGuarantees family=A region=aa8bc3911d985a299e88f859af353050] 2024-11-13T18:25:45,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/5caa9813a2ce4359a898e16d6b9f52d8 is 175, key is test_row_0/A:col10/1731522344530/Put/seqid=0 2024-11-13T18:25:45,474 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/2951a1be82d24fa0841076a8e4c88f7a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/2951a1be82d24fa0841076a8e4c88f7a 2024-11-13T18:25:45,483 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/A of aa8bc3911d985a299e88f859af353050 into c4184599f81848dd95f2b3e0c4f67c1c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:45,483 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:45,483 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/A, priority=13, startTime=1731522345174; duration=0sec 2024-11-13T18:25:45,483 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:45,483 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:A 2024-11-13T18:25:45,485 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa8bc3911d985a299e88f859af353050/C of aa8bc3911d985a299e88f859af353050 into 2951a1be82d24fa0841076a8e4c88f7a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:45,486 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:45,486 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050., storeName=aa8bc3911d985a299e88f859af353050/C, priority=13, startTime=1731522345174; duration=0sec 2024-11-13T18:25:45,487 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:45,487 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa8bc3911d985a299e88f859af353050:C 2024-11-13T18:25:45,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742265_1441 (size=31255) 2024-11-13T18:25:45,903 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/5caa9813a2ce4359a898e16d6b9f52d8 2024-11-13T18:25:45,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/5316c3fcf9b44aed9ab85459ada8451f is 50, key is test_row_0/B:col10/1731522344530/Put/seqid=0 2024-11-13T18:25:45,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742266_1442 (size=12301) 2024-11-13T18:25:45,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/5316c3fcf9b44aed9ab85459ada8451f 2024-11-13T18:25:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:25:45,946 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 630 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1870 rows 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 619 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1857 rows 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 631 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1879 rows 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 630 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1890 rows 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 631 2024-11-13T18:25:45,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1893 rows 2024-11-13T18:25:45,946 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:25:45,946 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:25:45,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:45,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:25:45,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:25:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:25:45,954 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522345954"}]},"ts":"1731522345954"} 2024-11-13T18:25:45,956 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:25:45,956 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:25:45,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:25:45,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, UNASSIGN}] 2024-11-13T18:25:45,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/de02bdedbf9043e6890b8df5d05e872a is 50, key is test_row_0/C:col10/1731522344530/Put/seqid=0 2024-11-13T18:25:45,961 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, UNASSIGN 2024-11-13T18:25:45,961 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=aa8bc3911d985a299e88f859af353050, regionState=CLOSING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:25:45,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, UNASSIGN because future has completed 2024-11-13T18:25:45,968 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:25:45,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:25:45,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742267_1443 (size=12301) 2024-11-13T18:25:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:25:46,121 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(122): Close aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:46,121 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:25:46,121 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1722): Closing aa8bc3911d985a299e88f859af353050, disabling compactions & flushes 2024-11-13T18:25:46,121 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:25:46,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/de02bdedbf9043e6890b8df5d05e872a 2024-11-13T18:25:46,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/A/5caa9813a2ce4359a898e16d6b9f52d8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5caa9813a2ce4359a898e16d6b9f52d8 2024-11-13T18:25:46,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5caa9813a2ce4359a898e16d6b9f52d8, entries=150, sequenceid=333, filesize=30.5 K 2024-11-13T18:25:46,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/B/5316c3fcf9b44aed9ab85459ada8451f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5316c3fcf9b44aed9ab85459ada8451f 2024-11-13T18:25:46,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5316c3fcf9b44aed9ab85459ada8451f, entries=150, sequenceid=333, filesize=12.0 K 2024-11-13T18:25:46,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/.tmp/C/de02bdedbf9043e6890b8df5d05e872a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/de02bdedbf9043e6890b8df5d05e872a 2024-11-13T18:25:46,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/de02bdedbf9043e6890b8df5d05e872a, entries=150, sequenceid=333, filesize=12.0 K 2024-11-13T18:25:46,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for aa8bc3911d985a299e88f859af353050 in 1184ms, sequenceid=333, compaction requested=false 2024-11-13T18:25:46,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa8bc3911d985a299e88f859af353050: 2024-11-13T18:25:46,416 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:46,416 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:46,416 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. after waiting 0 ms 2024-11-13T18:25:46,416 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:46,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/269bf68e36594331a02ab80e193687f5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/b41e6e51caed4625ac67f459379d40e8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/91b4d300aaed434da024c12f395c33eb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/a785d518751d48f194a22cf2be893bdc, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/2a2407a1e51147af9cede586c366317a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/1ee76691050641de91a76bdfa96bbbe7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/51c4c78bf0ad432bb1a371aa6a1b161f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/c53bad85e2f046cfbff89897716c358c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/bacf4726792d43acbee1d761758b1ec9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/72c4e1289b644799b96e75136615d055, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/130b089406bc40c788ad7d40a3b61e90, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/6ee8fbc5471b4ec79a41f989c317961c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/10ef0f261b5240759ff71b779793717e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/904f621d00e2446283a92491fd4e4e93, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/d82a23d9df854a95bd0750479cb7c605, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5e3ed0713a0d4855a7eac1dd72ebcd4f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/8cedf8866a6242d0a0a4be3dc40422ac, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/ae5eec4e71b34dd88f9b79f717e21bc6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/7c326117762a4b27bb11897b352fd425, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/4c1ed9661bc343f18877edc5e0372ba0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/19180115f55c4a9297dba10e84c9b988, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/65343a03422d41d2aeb0d66b6c64f74f] to archive 2024-11-13T18:25:46,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:46,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/269bf68e36594331a02ab80e193687f5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/269bf68e36594331a02ab80e193687f5 2024-11-13T18:25:46,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/b41e6e51caed4625ac67f459379d40e8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/b41e6e51caed4625ac67f459379d40e8 2024-11-13T18:25:46,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/91b4d300aaed434da024c12f395c33eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/91b4d300aaed434da024c12f395c33eb 2024-11-13T18:25:46,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/a785d518751d48f194a22cf2be893bdc to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/a785d518751d48f194a22cf2be893bdc 2024-11-13T18:25:46,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/2a2407a1e51147af9cede586c366317a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/2a2407a1e51147af9cede586c366317a 2024-11-13T18:25:46,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/1ee76691050641de91a76bdfa96bbbe7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/1ee76691050641de91a76bdfa96bbbe7 2024-11-13T18:25:46,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/51c4c78bf0ad432bb1a371aa6a1b161f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/51c4c78bf0ad432bb1a371aa6a1b161f 2024-11-13T18:25:46,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/c53bad85e2f046cfbff89897716c358c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/c53bad85e2f046cfbff89897716c358c 2024-11-13T18:25:46,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/bacf4726792d43acbee1d761758b1ec9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/bacf4726792d43acbee1d761758b1ec9 2024-11-13T18:25:46,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/72c4e1289b644799b96e75136615d055 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/72c4e1289b644799b96e75136615d055 2024-11-13T18:25:46,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/130b089406bc40c788ad7d40a3b61e90 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/130b089406bc40c788ad7d40a3b61e90 2024-11-13T18:25:46,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/6ee8fbc5471b4ec79a41f989c317961c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/6ee8fbc5471b4ec79a41f989c317961c 2024-11-13T18:25:46,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/10ef0f261b5240759ff71b779793717e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/10ef0f261b5240759ff71b779793717e 2024-11-13T18:25:46,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/904f621d00e2446283a92491fd4e4e93 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/904f621d00e2446283a92491fd4e4e93 2024-11-13T18:25:46,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/d82a23d9df854a95bd0750479cb7c605 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/d82a23d9df854a95bd0750479cb7c605 2024-11-13T18:25:46,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5e3ed0713a0d4855a7eac1dd72ebcd4f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5e3ed0713a0d4855a7eac1dd72ebcd4f 2024-11-13T18:25:46,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/8cedf8866a6242d0a0a4be3dc40422ac to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/8cedf8866a6242d0a0a4be3dc40422ac 2024-11-13T18:25:46,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/ae5eec4e71b34dd88f9b79f717e21bc6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/ae5eec4e71b34dd88f9b79f717e21bc6 2024-11-13T18:25:46,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/7c326117762a4b27bb11897b352fd425 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/7c326117762a4b27bb11897b352fd425 2024-11-13T18:25:46,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/4c1ed9661bc343f18877edc5e0372ba0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/4c1ed9661bc343f18877edc5e0372ba0 2024-11-13T18:25:46,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/19180115f55c4a9297dba10e84c9b988 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/19180115f55c4a9297dba10e84c9b988 2024-11-13T18:25:46,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/65343a03422d41d2aeb0d66b6c64f74f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/65343a03422d41d2aeb0d66b6c64f74f 2024-11-13T18:25:46,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/790194f8f26c4514bebd0af5d7d67889, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f58e43ee1782442196c17b0ae48aa832, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f612e2cca7764aa1926f603cb6bea22f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/2521a33dfa50474a862302c3053b645f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/9240a5d8d6774add90f8a47d4a243773, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/657693235d4846f0976c4e152c2120d4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e24c0ba1fa9d4b81893756695bc8668e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4c41580a4bfa4f4487870ae502005053, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e6128e7a18a44140b3cec4e787f14458, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/02ee9814ad1a4a029b964a36b1b29133, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/543abc6128704e2ba65a98794a8635ac, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/48968cb074884398bcd8508c9dc2f8f0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e92111bcda4642ddb261798181441515, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4632037eaf394e43aa49fe9832e8f904, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/0ff551856280448485b63959f50f025b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f6e64bb64b87424484f620704cd836dc, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5e80b399dada421e8237cdf838a81552, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5375e115574042c087fe1b36a4ee2193, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/cc930f003b41423991fe053f51e3ac27, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/c60757e9032149ed859769a4f225e2d1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/bb76a9606bcf4e5d863fbc4e3c418941, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e50d9dd4e86645478ca80060ce59a3db] to archive 2024-11-13T18:25:46,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:46,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/790194f8f26c4514bebd0af5d7d67889 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/790194f8f26c4514bebd0af5d7d67889 2024-11-13T18:25:46,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f58e43ee1782442196c17b0ae48aa832 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f58e43ee1782442196c17b0ae48aa832 2024-11-13T18:25:46,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f612e2cca7764aa1926f603cb6bea22f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f612e2cca7764aa1926f603cb6bea22f 2024-11-13T18:25:46,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/2521a33dfa50474a862302c3053b645f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/2521a33dfa50474a862302c3053b645f 2024-11-13T18:25:46,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/9240a5d8d6774add90f8a47d4a243773 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/9240a5d8d6774add90f8a47d4a243773 2024-11-13T18:25:46,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/657693235d4846f0976c4e152c2120d4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/657693235d4846f0976c4e152c2120d4 2024-11-13T18:25:46,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e24c0ba1fa9d4b81893756695bc8668e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e24c0ba1fa9d4b81893756695bc8668e 2024-11-13T18:25:46,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4c41580a4bfa4f4487870ae502005053 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4c41580a4bfa4f4487870ae502005053 2024-11-13T18:25:46,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e6128e7a18a44140b3cec4e787f14458 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e6128e7a18a44140b3cec4e787f14458 2024-11-13T18:25:46,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/02ee9814ad1a4a029b964a36b1b29133 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/02ee9814ad1a4a029b964a36b1b29133 2024-11-13T18:25:46,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/543abc6128704e2ba65a98794a8635ac to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/543abc6128704e2ba65a98794a8635ac 2024-11-13T18:25:46,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/48968cb074884398bcd8508c9dc2f8f0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/48968cb074884398bcd8508c9dc2f8f0 2024-11-13T18:25:46,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e92111bcda4642ddb261798181441515 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e92111bcda4642ddb261798181441515 2024-11-13T18:25:46,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4632037eaf394e43aa49fe9832e8f904 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/4632037eaf394e43aa49fe9832e8f904 2024-11-13T18:25:46,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/0ff551856280448485b63959f50f025b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/0ff551856280448485b63959f50f025b 2024-11-13T18:25:46,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f6e64bb64b87424484f620704cd836dc to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/f6e64bb64b87424484f620704cd836dc 2024-11-13T18:25:46,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5e80b399dada421e8237cdf838a81552 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5e80b399dada421e8237cdf838a81552 2024-11-13T18:25:46,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5375e115574042c087fe1b36a4ee2193 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5375e115574042c087fe1b36a4ee2193 2024-11-13T18:25:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:25:46,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/cc930f003b41423991fe053f51e3ac27 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/cc930f003b41423991fe053f51e3ac27 2024-11-13T18:25:46,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/c60757e9032149ed859769a4f225e2d1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/c60757e9032149ed859769a4f225e2d1 2024-11-13T18:25:46,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/bb76a9606bcf4e5d863fbc4e3c418941 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/bb76a9606bcf4e5d863fbc4e3c418941 2024-11-13T18:25:46,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e50d9dd4e86645478ca80060ce59a3db to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/e50d9dd4e86645478ca80060ce59a3db 2024-11-13T18:25:46,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01b6e654edf6436abd310df538944588, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/3d94a76624034349b5e226d9303c0789, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/89b82b471e3e4276b102b8e791a7a2d6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/a878e8e218d34e34993a5bedb0d17866, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/268e0d0b140a4e32aef185313466593e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/6a2fcc74356f43b3a21c4a6b0e5895a9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/dc8f3bb03207418fb082550d7043c53f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/baa29936c4784a88966bff6332608f19, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/93437ae6e2ca48c18073f379d7a0e366, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/817db45c8d424a6eb66a199b81730290, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8bdb079cd04c43bdac5efb72003b8c12, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/455bd6592ee74e818aa84a52cdedc1dc, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/cb68db0b821d4885a1e19320dd34f18c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/633a090b6774457292a49e7d93e19c4d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/411f1c0cf7fb424a8697b25c47f04a2e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01dd6ecd58fa40fe84febdef0f73d02c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8b8e6c9e929041838c41c5ca8371225e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/b671e7ba8b284fa593bb062daccebf9a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/487367d1c0434888828e31e3c0b68ee2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/52c18f2f270e4299bfa010e93dd0c1cb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/4da9092c5f6245f0a81b840ab3b154e2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/62b4cac5f09041f493094381cf36612a] to archive 2024-11-13T18:25:46,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:46,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01b6e654edf6436abd310df538944588 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01b6e654edf6436abd310df538944588 2024-11-13T18:25:46,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/3d94a76624034349b5e226d9303c0789 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/3d94a76624034349b5e226d9303c0789 2024-11-13T18:25:46,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/89b82b471e3e4276b102b8e791a7a2d6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/89b82b471e3e4276b102b8e791a7a2d6 2024-11-13T18:25:46,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/a878e8e218d34e34993a5bedb0d17866 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/a878e8e218d34e34993a5bedb0d17866 2024-11-13T18:25:46,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/268e0d0b140a4e32aef185313466593e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/268e0d0b140a4e32aef185313466593e 2024-11-13T18:25:46,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/6a2fcc74356f43b3a21c4a6b0e5895a9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/6a2fcc74356f43b3a21c4a6b0e5895a9 2024-11-13T18:25:46,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/dc8f3bb03207418fb082550d7043c53f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/dc8f3bb03207418fb082550d7043c53f 2024-11-13T18:25:46,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/baa29936c4784a88966bff6332608f19 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/baa29936c4784a88966bff6332608f19 2024-11-13T18:25:46,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/93437ae6e2ca48c18073f379d7a0e366 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/93437ae6e2ca48c18073f379d7a0e366 2024-11-13T18:25:46,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/817db45c8d424a6eb66a199b81730290 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/817db45c8d424a6eb66a199b81730290 2024-11-13T18:25:46,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8bdb079cd04c43bdac5efb72003b8c12 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8bdb079cd04c43bdac5efb72003b8c12 2024-11-13T18:25:46,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/455bd6592ee74e818aa84a52cdedc1dc to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/455bd6592ee74e818aa84a52cdedc1dc 2024-11-13T18:25:46,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/cb68db0b821d4885a1e19320dd34f18c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/cb68db0b821d4885a1e19320dd34f18c 2024-11-13T18:25:46,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/633a090b6774457292a49e7d93e19c4d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/633a090b6774457292a49e7d93e19c4d 2024-11-13T18:25:46,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/411f1c0cf7fb424a8697b25c47f04a2e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/411f1c0cf7fb424a8697b25c47f04a2e 2024-11-13T18:25:46,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01dd6ecd58fa40fe84febdef0f73d02c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/01dd6ecd58fa40fe84febdef0f73d02c 2024-11-13T18:25:46,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8b8e6c9e929041838c41c5ca8371225e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/8b8e6c9e929041838c41c5ca8371225e 2024-11-13T18:25:46,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/b671e7ba8b284fa593bb062daccebf9a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/b671e7ba8b284fa593bb062daccebf9a 2024-11-13T18:25:46,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/487367d1c0434888828e31e3c0b68ee2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/487367d1c0434888828e31e3c0b68ee2 2024-11-13T18:25:46,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/52c18f2f270e4299bfa010e93dd0c1cb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/52c18f2f270e4299bfa010e93dd0c1cb 2024-11-13T18:25:46,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/4da9092c5f6245f0a81b840ab3b154e2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/4da9092c5f6245f0a81b840ab3b154e2 2024-11-13T18:25:46,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/62b4cac5f09041f493094381cf36612a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/62b4cac5f09041f493094381cf36612a 2024-11-13T18:25:46,799 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/recovered.edits/339.seqid, newMaxSeqId=339, maxSeqId=4 2024-11-13T18:25:46,800 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050. 2024-11-13T18:25:46,800 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1676): Region close journal for aa8bc3911d985a299e88f859af353050: Waiting for close lock at 1731522346121Running coprocessor pre-close hooks at 1731522346121Disabling compacts and flushes for region at 1731522346121Disabling writes for close at 1731522346416 (+295 ms)Writing region close event to WAL at 1731522346782 (+366 ms)Running coprocessor post-close hooks at 1731522346800 (+18 ms)Closed at 1731522346800 2024-11-13T18:25:46,802 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(157): Closed aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:46,803 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=aa8bc3911d985a299e88f859af353050, regionState=CLOSED 2024-11-13T18:25:46,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:25:46,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-13T18:25:46,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseRegionProcedure aa8bc3911d985a299e88f859af353050, server=62ae331128c9,39615,1731522225114 in 837 msec 2024-11-13T18:25:46,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-13T18:25:46,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa8bc3911d985a299e88f859af353050, UNASSIGN in 849 msec 2024-11-13T18:25:46,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-13T18:25:46,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 853 msec 2024-11-13T18:25:46,815 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522346815"}]},"ts":"1731522346815"} 2024-11-13T18:25:46,816 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:25:46,816 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:25:46,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 870 msec 2024-11-13T18:25:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:25:47,087 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:47,088 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:25:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:47,099 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:25:47,101 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=103, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:47,115 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,133 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/recovered.edits] 2024-11-13T18:25:47,147 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5caa9813a2ce4359a898e16d6b9f52d8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/5caa9813a2ce4359a898e16d6b9f52d8 2024-11-13T18:25:47,164 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/c4184599f81848dd95f2b3e0c4f67c1c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/A/c4184599f81848dd95f2b3e0c4f67c1c 2024-11-13T18:25:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:25:47,214 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5316c3fcf9b44aed9ab85459ada8451f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/5316c3fcf9b44aed9ab85459ada8451f 2024-11-13T18:25:47,230 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/87621b9911ba45fa802f8aeecef21611 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/B/87621b9911ba45fa802f8aeecef21611 2024-11-13T18:25:47,242 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/2951a1be82d24fa0841076a8e4c88f7a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/2951a1be82d24fa0841076a8e4c88f7a 2024-11-13T18:25:47,248 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/de02bdedbf9043e6890b8df5d05e872a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/C/de02bdedbf9043e6890b8df5d05e872a 2024-11-13T18:25:47,264 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/recovered.edits/339.seqid to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050/recovered.edits/339.seqid 2024-11-13T18:25:47,270 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,270 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:25:47,271 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:25:47,277 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T18:25:47,286 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113038df1d3cce2483fad3b358a9c1388f7_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113038df1d3cce2483fad3b358a9c1388f7_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,295 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130ece74561571410cbc6056269688f3d1_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130ece74561571410cbc6056269688f3d1_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,310 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132549cbc2e0044b2ca4974f7d68d62869_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132549cbc2e0044b2ca4974f7d68d62869_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,316 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c2b99cd80334fb2a88e81a4e1d3e11c_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c2b99cd80334fb2a88e81a4e1d3e11c_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,318 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133b943de2606447bdabb074155d74e2ad_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133b943de2606447bdabb074155d74e2ad_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,320 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133bd54c6d98804b8db0d67d5830326dee_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133bd54c6d98804b8db0d67d5830326dee_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,321 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134448e71455784134bab21cef854cb066_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134448e71455784134bab21cef854cb066_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,323 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136425068a3ae94c489e37b8de8ff828ec_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136425068a3ae94c489e37b8de8ff828ec_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,324 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136b2bbc7aa6cc430da0d71d64d9e9212e_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136b2bbc7aa6cc430da0d71d64d9e9212e_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,327 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137a531d8792a34de794bba377f1055679_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137a531d8792a34de794bba377f1055679_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,329 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137ff15c4e886a412dbd8e805fbf148088_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137ff15c4e886a412dbd8e805fbf148088_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,332 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111389cad3aa41ff4ef9bf6170bc1eb6d954_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111389cad3aa41ff4ef9bf6170bc1eb6d954_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,334 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138dc2eac80ff84221b82e5deef8f068b6_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138dc2eac80ff84221b82e5deef8f068b6_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,338 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a71e1d775bda43f0b4b31c9a51fd64f8_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a71e1d775bda43f0b4b31c9a51fd64f8_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,354 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c4e413c9ae884aa8bb917f3bc3794881_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c4e413c9ae884aa8bb917f3bc3794881_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,359 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e0e9d8fc9f014534b1af9e2639ea072b_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e0e9d8fc9f014534b1af9e2639ea072b_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,362 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e2736d9b35164b269a6e5ce24ff4ea59_aa8bc3911d985a299e88f859af353050 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e2736d9b35164b269a6e5ce24ff4ea59_aa8bc3911d985a299e88f859af353050 2024-11-13T18:25:47,363 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:25:47,369 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=103, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:47,372 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:25:47,376 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:25:47,377 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=103, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:47,377 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:25:47,378 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731522347377"}]},"ts":"9223372036854775807"} 2024-11-13T18:25:47,381 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:25:47,381 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => aa8bc3911d985a299e88f859af353050, NAME => 'TestAcidGuarantees,,1731522322414.aa8bc3911d985a299e88f859af353050.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:25:47,381 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:25:47,381 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731522347381"}]},"ts":"9223372036854775807"} 2024-11-13T18:25:47,384 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:25:47,385 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=103, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:47,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 298 msec 2024-11-13T18:25:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:25:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:25:47,427 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:47,455 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 250), OpenFileDescriptor=463 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1955 (was 2083), ProcessCount=11 (was 11), AvailableMemoryMB=1786 (was 1540) - AvailableMemoryMB LEAK? - 2024-11-13T18:25:47,488 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=1955, ProcessCount=11, AvailableMemoryMB=1780 2024-11-13T18:25:47,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:25:47,495 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:25:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:25:47,500 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:25:47,501 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:47,501 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 104 2024-11-13T18:25:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T18:25:47,502 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:25:47,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742268_1444 (size=1061) 2024-11-13T18:25:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T18:25:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T18:25:47,976 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b 2024-11-13T18:25:48,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742269_1445 (size=53) 2024-11-13T18:25:48,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:48,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ecb8a98d59975d334af6c23fb4444cd4, disabling compactions & flushes 2024-11-13T18:25:48,071 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:48,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:48,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. after waiting 0 ms 2024-11-13T18:25:48,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:48,071 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:48,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ecb8a98d59975d334af6c23fb4444cd4: Waiting for close lock at 1731522348071Disabling compacts and flushes for region at 1731522348071Disabling writes for close at 1731522348071Writing region close event to WAL at 1731522348071Closed at 1731522348071 2024-11-13T18:25:48,073 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:25:48,073 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731522348073"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731522348073"}]},"ts":"1731522348073"} 2024-11-13T18:25:48,076 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:25:48,087 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:25:48,087 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522348087"}]},"ts":"1731522348087"} 2024-11-13T18:25:48,098 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:25:48,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ecb8a98d59975d334af6c23fb4444cd4, ASSIGN}] 2024-11-13T18:25:48,107 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ecb8a98d59975d334af6c23fb4444cd4, ASSIGN 2024-11-13T18:25:48,110 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ecb8a98d59975d334af6c23fb4444cd4, ASSIGN; state=OFFLINE, location=62ae331128c9,39615,1731522225114; forceNewPlan=false, retain=false 2024-11-13T18:25:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T18:25:48,262 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=ecb8a98d59975d334af6c23fb4444cd4, regionState=OPENING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:25:48,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ecb8a98d59975d334af6c23fb4444cd4, ASSIGN because future has completed 2024-11-13T18:25:48,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:25:48,425 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:48,425 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:48,426 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,426 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:48,426 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,426 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,432 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,435 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:25:48,435 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecb8a98d59975d334af6c23fb4444cd4 columnFamilyName A 2024-11-13T18:25:48,435 DEBUG [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:48,436 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] regionserver.HStore(327): Store=ecb8a98d59975d334af6c23fb4444cd4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:48,436 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,441 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:25:48,442 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecb8a98d59975d334af6c23fb4444cd4 columnFamilyName B 2024-11-13T18:25:48,442 DEBUG [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:48,442 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] regionserver.HStore(327): Store=ecb8a98d59975d334af6c23fb4444cd4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:48,442 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,444 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:25:48,444 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecb8a98d59975d334af6c23fb4444cd4 columnFamilyName C 2024-11-13T18:25:48,444 DEBUG [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:48,445 INFO [StoreOpener-ecb8a98d59975d334af6c23fb4444cd4-1 {}] regionserver.HStore(327): Store=ecb8a98d59975d334af6c23fb4444cd4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:48,445 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:48,445 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,446 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,446 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,448 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,448 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,448 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:25:48,451 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,454 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:48,454 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened ecb8a98d59975d334af6c23fb4444cd4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66579178, jitterRate=-0.00789293646812439}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:25:48,455 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,455 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for ecb8a98d59975d334af6c23fb4444cd4: Running coprocessor pre-open hook at 1731522348426Writing region info on filesystem at 1731522348426Initializing all the Stores at 1731522348428 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522348428Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522348432 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522348432Cleaning up temporary data from old regions at 1731522348448 (+16 ms)Running coprocessor post-open hooks at 1731522348455 (+7 ms)Region opened successfully at 1731522348455 2024-11-13T18:25:48,457 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., pid=106, masterSystemTime=1731522348418 2024-11-13T18:25:48,467 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:48,467 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:48,471 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=ecb8a98d59975d334af6c23fb4444cd4, regionState=OPEN, openSeqNum=2, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:25:48,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:25:48,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-13T18:25:48,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; OpenRegionProcedure ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 in 212 msec 2024-11-13T18:25:48,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-13T18:25:48,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ecb8a98d59975d334af6c23fb4444cd4, ASSIGN in 381 msec 2024-11-13T18:25:48,483 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:25:48,484 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522348483"}]},"ts":"1731522348483"} 2024-11-13T18:25:48,492 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:25:48,493 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:25:48,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 997 msec 2024-11-13T18:25:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T18:25:48,637 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:48,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d863fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,641 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42438f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,645 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d05e79b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,649 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,653 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:25:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 62ae331128c9,45483,1731522221514 2024-11-13T18:25:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1be32976 2024-11-13T18:25:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:25:48,658 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:25:48,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b94fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,664 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17c595b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,666 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10b50017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53dea8b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,675 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fd170da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c86dedc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733fcf91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,685 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ea9330, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,688 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e60daaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5921fce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,695 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e2d44b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,701 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36fce6c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b52d1e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,716 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa1f916, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,718 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b3862f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d3fce0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,725 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17852b30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,728 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b7d128, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337c72b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,738 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c4f9079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,741 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@124d19f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebb75cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,753 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26024c4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,755 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eea21e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be17334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,760 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6abac302, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,761 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302f7a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6302c151, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:25:48,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:48,763 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:25:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:25:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7409c83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:25:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:48,764 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93195a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:48,767 DEBUG [Thread-2019 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:25:48,772 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-13T18:25:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:25:48,775 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:48,775 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:48,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:48,779 DEBUG [Thread-2002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:25:48,791 DEBUG [Thread-2008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:48,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:48,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:48,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:48,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:48,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:48,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:48,807 DEBUG [Thread-2021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,808 DEBUG [Thread-2021 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,809 DEBUG [Thread-2021 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,810 DEBUG [Thread-2015 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:25:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:25:48,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:25:48,828 DEBUG [Thread-2006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,829 DEBUG [Thread-2006 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,830 DEBUG [Thread-2006 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:25:48,835 DEBUG [Thread-2004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:25:48,844 DEBUG [Thread-2010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,847 DEBUG [Thread-2017 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:25:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:25:48,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/a1fe25f47f94458ca884d39cc502fde2 is 50, key is test_row_0/A:col10/1731522348789/Put/seqid=0 2024-11-13T18:25:48,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731522348839, max timestamp=1731522348857Num uniques 859; 2024-11-13T18:25:48,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-13T18:25:48,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731522348839, max timestamp=1731522348857Num uniques 859; 2024-11-13T18:25:48,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-13T18:25:48,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731522348839, max timestamp=1731522348857Num uniques 859; 2024-11-13T18:25:48,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-13T18:25:48,871 DEBUG [Thread-2013 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:25:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:48,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2] 2024-11-13T18:25:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:25:48,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742270_1446 (size=12001) 2024-11-13T18:25:48,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/a1fe25f47f94458ca884d39cc502fde2 2024-11-13T18:25:48,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522408892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:48,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:48,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:48,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:48,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:48,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:48,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:48,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522408912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:48,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522408914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:48,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522408917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:48,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522408919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:48,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/b18e8d487a69482599ca256b4091c42a is 50, key is test_row_0/B:col10/1731522348789/Put/seqid=0 2024-11-13T18:25:49,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742271_1447 (size=12001) 2024-11-13T18:25:49,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522409026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522409036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522409047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522409047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522409047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:49,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:49,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:25:49,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522409235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522409256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522409259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522409259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522409259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:49,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:49,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:25:49,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/b18e8d487a69482599ca256b4091c42a 2024-11-13T18:25:49,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/e9c69ed7851e49b5807fcf5419ecb312 is 50, key is test_row_0/C:col10/1731522348789/Put/seqid=0 2024-11-13T18:25:49,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742272_1448 (size=12001) 2024-11-13T18:25:49,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/e9c69ed7851e49b5807fcf5419ecb312 2024-11-13T18:25:49,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/a1fe25f47f94458ca884d39cc502fde2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/a1fe25f47f94458ca884d39cc502fde2 2024-11-13T18:25:49,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/a1fe25f47f94458ca884d39cc502fde2, entries=150, sequenceid=11, filesize=11.7 K 2024-11-13T18:25:49,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/b18e8d487a69482599ca256b4091c42a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b18e8d487a69482599ca256b4091c42a 2024-11-13T18:25:49,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b18e8d487a69482599ca256b4091c42a, entries=150, sequenceid=11, filesize=11.7 K 2024-11-13T18:25:49,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/e9c69ed7851e49b5807fcf5419ecb312 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e9c69ed7851e49b5807fcf5419ecb312 2024-11-13T18:25:49,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e9c69ed7851e49b5807fcf5419ecb312, entries=150, sequenceid=11, filesize=11.7 K 2024-11-13T18:25:49,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ecb8a98d59975d334af6c23fb4444cd4 in 760ms, sequenceid=11, compaction requested=false 2024-11-13T18:25:49,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:49,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=147.60 KB heapSize=401.39 KB 2024-11-13T18:25:49,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:49,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:49,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:49,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:49,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:49,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:49,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/2652c0462bac44feb9468bbcead2ced1 is 50, key is test_row_0/A:col10/1731522348910/Put/seqid=0 2024-11-13T18:25:49,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522409607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522409609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522409611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742273_1449 (size=14341) 2024-11-13T18:25:49,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/2652c0462bac44feb9468bbcead2ced1 2024-11-13T18:25:49,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522409627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522409627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/ccff7c897fca4497b78e558f37f8e2fa is 50, key is test_row_0/B:col10/1731522348910/Put/seqid=0 2024-11-13T18:25:49,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742274_1450 (size=12001) 2024-11-13T18:25:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/ccff7c897fca4497b78e558f37f8e2fa 2024-11-13T18:25:49,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522409736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522409738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522409739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/139c1f470b524532ac62b809b445293b is 50, key is test_row_0/C:col10/1731522348910/Put/seqid=0 2024-11-13T18:25:49,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522409766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522409766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742275_1451 (size=12001) 2024-11-13T18:25:49,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:49,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:49,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:49,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:25:49,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522409956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522409966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:49,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522409971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522409996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522409999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:50,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:50,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/139c1f470b524532ac62b809b445293b 2024-11-13T18:25:50,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/2652c0462bac44feb9468bbcead2ced1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2652c0462bac44feb9468bbcead2ced1 2024-11-13T18:25:50,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2652c0462bac44feb9468bbcead2ced1, entries=200, sequenceid=45, filesize=14.0 K 2024-11-13T18:25:50,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/ccff7c897fca4497b78e558f37f8e2fa as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/ccff7c897fca4497b78e558f37f8e2fa 2024-11-13T18:25:50,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/ccff7c897fca4497b78e558f37f8e2fa, entries=150, sequenceid=45, filesize=11.7 K 2024-11-13T18:25:50,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/139c1f470b524532ac62b809b445293b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/139c1f470b524532ac62b809b445293b 2024-11-13T18:25:50,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522410281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522410288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/139c1f470b524532ac62b809b445293b, entries=150, sequenceid=45, filesize=11.7 K 2024-11-13T18:25:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~401.34 KB/410976, currentSize=46.96 KB/48090 for ecb8a98d59975d334af6c23fb4444cd4 in 714ms, sequenceid=45, compaction requested=false 2024-11-13T18:25:50,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:50,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T18:25:50,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:50,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:50,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:50,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:50,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:50,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:50,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/09d43651db364ecd9a4d8181d3cdbab0 is 50, key is test_row_0/A:col10/1731522349605/Put/seqid=0 2024-11-13T18:25:50,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742276_1452 (size=14341) 2024-11-13T18:25:50,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:50,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:50,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/09d43651db364ecd9a4d8181d3cdbab0 2024-11-13T18:25:50,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/8f7ebb0892234ef0aac385ce2336a604 is 50, key is test_row_0/B:col10/1731522349605/Put/seqid=0 2024-11-13T18:25:50,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742277_1453 (size=12001) 2024-11-13T18:25:50,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/8f7ebb0892234ef0aac385ce2336a604 2024-11-13T18:25:50,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522410468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/953a049cc36e481dbb8920dea50c066d is 50, key is test_row_0/C:col10/1731522349605/Put/seqid=0 2024-11-13T18:25:50,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522350421, max timestamp=1731522350499Num uniques 859; 2024-11-13T18:25:50,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:50,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522350421, max timestamp=1731522350499Num uniques 859; 2024-11-13T18:25:50,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:50,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:50,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731522350421, max timestamp=1731522350499Num uniques 859; 2024-11-13T18:25:50,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T18:25:50,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742278_1454 (size=12001) 2024-11-13T18:25:50,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522410650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:50,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522410674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522410676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522410786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522410805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522410806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522410806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522410807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:50,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:50,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:25:50,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/953a049cc36e481dbb8920dea50c066d 2024-11-13T18:25:50,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/09d43651db364ecd9a4d8181d3cdbab0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/09d43651db364ecd9a4d8181d3cdbab0 2024-11-13T18:25:50,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/09d43651db364ecd9a4d8181d3cdbab0, entries=200, sequenceid=56, filesize=14.0 K 2024-11-13T18:25:50,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/8f7ebb0892234ef0aac385ce2336a604 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8f7ebb0892234ef0aac385ce2336a604 2024-11-13T18:25:50,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8f7ebb0892234ef0aac385ce2336a604, entries=150, sequenceid=56, filesize=11.7 K 2024-11-13T18:25:50,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:50,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:50,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:50,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/953a049cc36e481dbb8920dea50c066d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/953a049cc36e481dbb8920dea50c066d 2024-11-13T18:25:50,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/953a049cc36e481dbb8920dea50c066d, entries=150, sequenceid=56, filesize=11.7 K 2024-11-13T18:25:51,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ecb8a98d59975d334af6c23fb4444cd4 in 711ms, sequenceid=56, compaction requested=true 2024-11-13T18:25:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:25:51,014 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:51,014 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:51,024 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:51,024 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/C is initiating minor compaction (all files) 2024-11-13T18:25:51,024 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/C in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,024 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e9c69ed7851e49b5807fcf5419ecb312, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/139c1f470b524532ac62b809b445293b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/953a049cc36e481dbb8920dea50c066d] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=35.2 K 2024-11-13T18:25:51,031 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:51,031 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/A is initiating minor compaction (all files) 2024-11-13T18:25:51,031 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/A in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,031 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/a1fe25f47f94458ca884d39cc502fde2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2652c0462bac44feb9468bbcead2ced1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/09d43651db364ecd9a4d8181d3cdbab0] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=39.7 K 2024-11-13T18:25:51,032 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1fe25f47f94458ca884d39cc502fde2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731522348789 2024-11-13T18:25:51,032 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2652c0462bac44feb9468bbcead2ced1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731522348910 2024-11-13T18:25:51,032 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09d43651db364ecd9a4d8181d3cdbab0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731522349605 2024-11-13T18:25:51,033 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e9c69ed7851e49b5807fcf5419ecb312, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731522348789 2024-11-13T18:25:51,039 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 139c1f470b524532ac62b809b445293b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731522348910 2024-11-13T18:25:51,042 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 953a049cc36e481dbb8920dea50c066d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731522349605 2024-11-13T18:25:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:51,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T18:25:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:51,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/0acd406385cd42eda83d05e65e4fbee3 is 50, key is test_row_0/A:col10/1731522350648/Put/seqid=0 2024-11-13T18:25:51,090 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#A#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:51,091 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/d9a2cd9aa0824e85afebfa410754255f is 50, key is test_row_0/A:col10/1731522349605/Put/seqid=0 2024-11-13T18:25:51,107 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:51,107 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/ed64d6261ef24b1780b68619cd628685 is 50, key is test_row_0/C:col10/1731522349605/Put/seqid=0 2024-11-13T18:25:51,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522411110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522411112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:51,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522411128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742280_1456 (size=12104) 2024-11-13T18:25:51,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742279_1455 (size=14341) 2024-11-13T18:25:51,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742281_1457 (size=12104) 2024-11-13T18:25:51,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522411245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522411256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522411271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:51,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:51,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:51,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:51,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522411476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522411476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522411499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/0acd406385cd42eda83d05e65e4fbee3 2024-11-13T18:25:51,628 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/d9a2cd9aa0824e85afebfa410754255f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/d9a2cd9aa0824e85afebfa410754255f 2024-11-13T18:25:51,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/948e952df50f464f932ff88e6fa264a0 is 50, key is test_row_0/B:col10/1731522350648/Put/seqid=0 2024-11-13T18:25:51,647 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/ed64d6261ef24b1780b68619cd628685 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/ed64d6261ef24b1780b68619cd628685 2024-11-13T18:25:51,653 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/A of ecb8a98d59975d334af6c23fb4444cd4 into d9a2cd9aa0824e85afebfa410754255f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:51,653 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:51,653 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/A, priority=13, startTime=1731522351009; duration=0sec 2024-11-13T18:25:51,654 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:51,654 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:A 2024-11-13T18:25:51,654 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:51,659 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/C of ecb8a98d59975d334af6c23fb4444cd4 into ed64d6261ef24b1780b68619cd628685(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:51,659 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:51,659 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/C, priority=13, startTime=1731522351010; duration=0sec 2024-11-13T18:25:51,659 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:51,659 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:C 2024-11-13T18:25:51,659 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:51,659 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/B is initiating minor compaction (all files) 2024-11-13T18:25:51,659 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/B in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,660 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b18e8d487a69482599ca256b4091c42a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/ccff7c897fca4497b78e558f37f8e2fa, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8f7ebb0892234ef0aac385ce2336a604] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=35.2 K 2024-11-13T18:25:51,660 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b18e8d487a69482599ca256b4091c42a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731522348789 2024-11-13T18:25:51,660 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccff7c897fca4497b78e558f37f8e2fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731522348910 2024-11-13T18:25:51,661 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f7ebb0892234ef0aac385ce2336a604, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731522349605 2024-11-13T18:25:51,682 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#B#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:51,682 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/9f02b34001df4c8d83a5071b0295731c is 50, key is test_row_0/B:col10/1731522349605/Put/seqid=0 2024-11-13T18:25:51,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742282_1458 (size=12001) 2024-11-13T18:25:51,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742283_1459 (size=12104) 2024-11-13T18:25:51,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/948e952df50f464f932ff88e6fa264a0 2024-11-13T18:25:51,750 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/9f02b34001df4c8d83a5071b0295731c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/9f02b34001df4c8d83a5071b0295731c 2024-11-13T18:25:51,761 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/B of ecb8a98d59975d334af6c23fb4444cd4 into 9f02b34001df4c8d83a5071b0295731c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:51,762 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:51,762 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/B, priority=13, startTime=1731522351010; duration=0sec 2024-11-13T18:25:51,762 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:51,762 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:B 2024-11-13T18:25:51,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/7939285d0032422abf0a5ba0fc8b2bee is 50, key is test_row_0/C:col10/1731522350648/Put/seqid=0 2024-11-13T18:25:51,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:51,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:51,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522411806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522411806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742284_1460 (size=12001) 2024-11-13T18:25:51,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522411838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522411846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,866 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in ecb8a98d59975d334af6c23fb4444cd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:25:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522411847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:51,885 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in ecb8a98d59975d334af6c23fb4444cd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:25:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:51,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/a1fe25f47f94458ca884d39cc502fde2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2652c0462bac44feb9468bbcead2ced1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/09d43651db364ecd9a4d8181d3cdbab0] to archive 2024-11-13T18:25:51,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:51,917 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/a1fe25f47f94458ca884d39cc502fde2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/a1fe25f47f94458ca884d39cc502fde2 2024-11-13T18:25:51,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2652c0462bac44feb9468bbcead2ced1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2652c0462bac44feb9468bbcead2ced1 2024-11-13T18:25:51,919 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/09d43651db364ecd9a4d8181d3cdbab0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/09d43651db364ecd9a4d8181d3cdbab0 2024-11-13T18:25:51,921 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b18e8d487a69482599ca256b4091c42a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/ccff7c897fca4497b78e558f37f8e2fa, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8f7ebb0892234ef0aac385ce2336a604] to archive 2024-11-13T18:25:51,922 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:51,927 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b18e8d487a69482599ca256b4091c42a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b18e8d487a69482599ca256b4091c42a 2024-11-13T18:25:51,929 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/ccff7c897fca4497b78e558f37f8e2fa to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/ccff7c897fca4497b78e558f37f8e2fa 2024-11-13T18:25:51,930 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8f7ebb0892234ef0aac385ce2336a604 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8f7ebb0892234ef0aac385ce2336a604 2024-11-13T18:25:51,932 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e9c69ed7851e49b5807fcf5419ecb312, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/139c1f470b524532ac62b809b445293b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/953a049cc36e481dbb8920dea50c066d] to archive 2024-11-13T18:25:51,932 WARN [regionserver/62ae331128c9:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 1, running: 1 2024-11-13T18:25:51,933 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:51,935 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e9c69ed7851e49b5807fcf5419ecb312 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e9c69ed7851e49b5807fcf5419ecb312 2024-11-13T18:25:51,936 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/139c1f470b524532ac62b809b445293b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/139c1f470b524532ac62b809b445293b 2024-11-13T18:25:51,941 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/62ae331128c9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/953a049cc36e481dbb8920dea50c066d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/953a049cc36e481dbb8920dea50c066d 2024-11-13T18:25:51,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:51,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:51,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:51,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:52,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:52,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:52,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:52,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:52,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:52,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:52,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/7939285d0032422abf0a5ba0fc8b2bee 2024-11-13T18:25:52,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/0acd406385cd42eda83d05e65e4fbee3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/0acd406385cd42eda83d05e65e4fbee3 2024-11-13T18:25:52,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/0acd406385cd42eda83d05e65e4fbee3, entries=200, sequenceid=92, filesize=14.0 K 2024-11-13T18:25:52,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/948e952df50f464f932ff88e6fa264a0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/948e952df50f464f932ff88e6fa264a0 2024-11-13T18:25:52,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/948e952df50f464f932ff88e6fa264a0, entries=150, sequenceid=92, filesize=11.7 K 2024-11-13T18:25:52,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/7939285d0032422abf0a5ba0fc8b2bee as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7939285d0032422abf0a5ba0fc8b2bee 2024-11-13T18:25:52,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7939285d0032422abf0a5ba0fc8b2bee, entries=150, sequenceid=92, filesize=11.7 K 2024-11-13T18:25:52,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T18:25:52,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ecb8a98d59975d334af6c23fb4444cd4 in 1223ms, sequenceid=92, compaction requested=false 2024-11-13T18:25:52,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:52,273 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:25:52,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:52,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:52,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:52,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:52,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:52,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:52,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/99e98b792fc64775a6f5a2146298654a is 50, key is test_row_0/A:col10/1731522351122/Put/seqid=0 2024-11-13T18:25:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:52,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:52,341 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:25:52,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742285_1461 (size=12001) 2024-11-13T18:25:52,348 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/99e98b792fc64775a6f5a2146298654a 2024-11-13T18:25:52,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/5d526a2679f04c06a0a533a5569f4357 is 50, key is test_row_0/B:col10/1731522351122/Put/seqid=0 2024-11-13T18:25:52,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742286_1462 (size=12001) 2024-11-13T18:25:52,435 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/5d526a2679f04c06a0a533a5569f4357 2024-11-13T18:25:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/3999ca816dc44200a2f06f72b85dbc12 is 50, key is test_row_0/C:col10/1731522351122/Put/seqid=0 2024-11-13T18:25:52,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742287_1463 (size=12001) 2024-11-13T18:25:52,497 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/3999ca816dc44200a2f06f72b85dbc12 2024-11-13T18:25:52,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/99e98b792fc64775a6f5a2146298654a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/99e98b792fc64775a6f5a2146298654a 2024-11-13T18:25:52,514 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/99e98b792fc64775a6f5a2146298654a, entries=150, sequenceid=105, filesize=11.7 K 2024-11-13T18:25:52,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/5d526a2679f04c06a0a533a5569f4357 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5d526a2679f04c06a0a533a5569f4357 2024-11-13T18:25:52,535 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5d526a2679f04c06a0a533a5569f4357, entries=150, sequenceid=105, filesize=11.7 K 2024-11-13T18:25:52,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/3999ca816dc44200a2f06f72b85dbc12 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/3999ca816dc44200a2f06f72b85dbc12 2024-11-13T18:25:52,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522352401, max timestamp=1731522352540Num uniques 859; 2024-11-13T18:25:52,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:52,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522352401, max timestamp=1731522352538Num uniques 859; 2024-11-13T18:25:52,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:52,555 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/3999ca816dc44200a2f06f72b85dbc12, entries=150, sequenceid=105, filesize=11.7 K 2024-11-13T18:25:52,556 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=71.56 KB/73280 for ecb8a98d59975d334af6c23fb4444cd4 in 284ms, sequenceid=105, compaction requested=true 2024-11-13T18:25:52,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:52,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:52,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-13T18:25:52,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522352401, max timestamp=1731522352538Num uniques 859; 2024-11-13T18:25:52,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-13T18:25:52,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-13T18:25:52,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7880 sec 2024-11-13T18:25:52,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.7950 sec 2024-11-13T18:25:52,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-11-13T18:25:52,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:52,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:52,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:52,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:52,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:52,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:52,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/b089e14d7cd5494fa07b11f82c6a04c0 is 50, key is test_row_0/A:col10/1731522352553/Put/seqid=0 2024-11-13T18:25:52,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742288_1464 (size=19171) 2024-11-13T18:25:52,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522412744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:52,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522352664, max timestamp=1731522352771Num uniques 859; 2024-11-13T18:25:52,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:52,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522352664, max timestamp=1731522352771Num uniques 859; 2024-11-13T18:25:52,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:52,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522352664, max timestamp=1731522352771Num uniques 859; 2024-11-13T18:25:52,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:25:52,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522412940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:25:52,956 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:52,956 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-13T18:25:52,958 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:52,963 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:52,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:25:52,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522412954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:52,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522412955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/b089e14d7cd5494fa07b11f82c6a04c0 2024-11-13T18:25:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:25:53,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522413066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/d8cf2dd856504bf4b3154c4dc6d39e07 is 50, key is test_row_0/B:col10/1731522352553/Put/seqid=0 2024-11-13T18:25:53,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T18:25:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:53,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:53,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522413096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522413096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:53,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742289_1465 (size=12051) 2024-11-13T18:25:53,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/d8cf2dd856504bf4b3154c4dc6d39e07 2024-11-13T18:25:53,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/2d6e85a1fdf94e41a86df5bb3b7c7739 is 50, key is test_row_0/C:col10/1731522352553/Put/seqid=0 2024-11-13T18:25:53,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742290_1466 (size=12051) 2024-11-13T18:25:53,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/2d6e85a1fdf94e41a86df5bb3b7c7739 2024-11-13T18:25:53,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:25:53,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T18:25:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:53,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:53,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522413298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/b089e14d7cd5494fa07b11f82c6a04c0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b089e14d7cd5494fa07b11f82c6a04c0 2024-11-13T18:25:53,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522413345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b089e14d7cd5494fa07b11f82c6a04c0, entries=300, sequenceid=131, filesize=18.7 K 2024-11-13T18:25:53,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522413348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/d8cf2dd856504bf4b3154c4dc6d39e07 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d8cf2dd856504bf4b3154c4dc6d39e07 2024-11-13T18:25:53,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d8cf2dd856504bf4b3154c4dc6d39e07, entries=150, sequenceid=131, filesize=11.8 K 2024-11-13T18:25:53,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/2d6e85a1fdf94e41a86df5bb3b7c7739 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2d6e85a1fdf94e41a86df5bb3b7c7739 2024-11-13T18:25:53,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2d6e85a1fdf94e41a86df5bb3b7c7739, entries=150, sequenceid=131, filesize=11.8 K 2024-11-13T18:25:53,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=127.47 KB/130530 for ecb8a98d59975d334af6c23fb4444cd4 in 845ms, sequenceid=131, compaction requested=true 2024-11-13T18:25:53,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:53,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:53,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:25:53,441 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:53,442 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:53,443 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57617 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:53,443 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/A is initiating minor compaction (all files) 2024-11-13T18:25:53,443 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/A in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:53,443 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/d9a2cd9aa0824e85afebfa410754255f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/0acd406385cd42eda83d05e65e4fbee3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/99e98b792fc64775a6f5a2146298654a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b089e14d7cd5494fa07b11f82c6a04c0] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=56.3 K 2024-11-13T18:25:53,449 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a2cd9aa0824e85afebfa410754255f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731522349605 2024-11-13T18:25:53,455 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 0acd406385cd42eda83d05e65e4fbee3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731522350648 2024-11-13T18:25:53,455 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:53,455 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/C is initiating minor compaction (all files) 2024-11-13T18:25:53,455 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/C in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:53,455 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/ed64d6261ef24b1780b68619cd628685, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7939285d0032422abf0a5ba0fc8b2bee, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/3999ca816dc44200a2f06f72b85dbc12, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2d6e85a1fdf94e41a86df5bb3b7c7739] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=47.0 K 2024-11-13T18:25:53,455 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 99e98b792fc64775a6f5a2146298654a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731522351079 2024-11-13T18:25:53,459 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed64d6261ef24b1780b68619cd628685, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731522349605 2024-11-13T18:25:53,459 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting b089e14d7cd5494fa07b11f82c6a04c0, keycount=300, bloomtype=ROW, size=18.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731522352401 2024-11-13T18:25:53,466 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7939285d0032422abf0a5ba0fc8b2bee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731522350648 2024-11-13T18:25:53,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T18:25:53,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:53,466 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-13T18:25:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:25:53,470 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3999ca816dc44200a2f06f72b85dbc12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731522351079 2024-11-13T18:25:53,478 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6e85a1fdf94e41a86df5bb3b7c7739, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731522352496 2024-11-13T18:25:53,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/4d34beb8d3fd459e8b5f18af156d7f43 is 50, key is test_row_0/A:col10/1731522352937/Put/seqid=0 2024-11-13T18:25:53,540 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#C#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:53,541 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/402947b313184853b682a2cd2552ed37 is 50, key is test_row_0/C:col10/1731522352553/Put/seqid=0 2024-11-13T18:25:53,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742291_1467 (size=12151) 2024-11-13T18:25:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:25:53,599 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#A#compaction#392 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:53,599 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/ba8b8eceab144060a4e4becfa84edc35 is 50, key is test_row_0/A:col10/1731522352553/Put/seqid=0 2024-11-13T18:25:53,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742292_1468 (size=12189) 2024-11-13T18:25:53,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742293_1469 (size=12189) 2024-11-13T18:25:53,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:53,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522413749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522413750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522413793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522413875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,900 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in ecb8a98d59975d334af6c23fb4444cd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:25:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522413906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,958 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in ecb8a98d59975d334af6c23fb4444cd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4349 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:25:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522413906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522413907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:53,979 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/4d34beb8d3fd459e8b5f18af156d7f43 2024-11-13T18:25:53,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522413956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:54,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/644f6bdc8aef4e41b0e21fd0ca4fa437 is 50, key is test_row_0/B:col10/1731522352937/Put/seqid=0 2024-11-13T18:25:54,012 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/402947b313184853b682a2cd2552ed37 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/402947b313184853b682a2cd2552ed37 2024-11-13T18:25:54,030 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/C of ecb8a98d59975d334af6c23fb4444cd4 into 402947b313184853b682a2cd2552ed37(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:54,030 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:54,030 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/C, priority=12, startTime=1731522353441; duration=0sec 2024-11-13T18:25:54,030 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:54,030 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:C 2024-11-13T18:25:54,031 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:25:54,034 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:25:54,034 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/B is initiating minor compaction (all files) 2024-11-13T18:25:54,034 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/B in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:54,035 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/9f02b34001df4c8d83a5071b0295731c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/948e952df50f464f932ff88e6fa264a0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5d526a2679f04c06a0a533a5569f4357, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d8cf2dd856504bf4b3154c4dc6d39e07] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=47.0 K 2024-11-13T18:25:54,036 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f02b34001df4c8d83a5071b0295731c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731522349605 2024-11-13T18:25:54,037 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 948e952df50f464f932ff88e6fa264a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731522350648 2024-11-13T18:25:54,039 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d526a2679f04c06a0a533a5569f4357, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731522351079 2024-11-13T18:25:54,041 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8cf2dd856504bf4b3154c4dc6d39e07, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731522352496 2024-11-13T18:25:54,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742294_1470 (size=12151) 2024-11-13T18:25:54,068 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/644f6bdc8aef4e41b0e21fd0ca4fa437 2024-11-13T18:25:54,074 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#B#compaction#394 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:54,074 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/69b074555d5e4514861382bf9a6890d1 is 50, key is test_row_0/B:col10/1731522352553/Put/seqid=0 2024-11-13T18:25:54,093 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/ba8b8eceab144060a4e4becfa84edc35 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/ba8b8eceab144060a4e4becfa84edc35 2024-11-13T18:25:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:25:54,108 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/A of ecb8a98d59975d334af6c23fb4444cd4 into ba8b8eceab144060a4e4becfa84edc35(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:54,108 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:54,108 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/A, priority=12, startTime=1731522353440; duration=0sec 2024-11-13T18:25:54,108 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:54,109 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:A 2024-11-13T18:25:54,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/0031cc22128d4d6bb25de2f5acc085c4 is 50, key is test_row_0/C:col10/1731522352937/Put/seqid=0 2024-11-13T18:25:54,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742295_1471 (size=12189) 2024-11-13T18:25:54,145 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/69b074555d5e4514861382bf9a6890d1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/69b074555d5e4514861382bf9a6890d1 2024-11-13T18:25:54,151 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/B of ecb8a98d59975d334af6c23fb4444cd4 into 69b074555d5e4514861382bf9a6890d1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:54,151 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:54,151 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/B, priority=12, startTime=1731522353441; duration=0sec 2024-11-13T18:25:54,151 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:54,151 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:B 2024-11-13T18:25:54,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522414166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522414166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:54,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742296_1472 (size=12151) 2024-11-13T18:25:54,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522414205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:54,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522414496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:54,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522414497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:54,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:54,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522414526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:54,594 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/0031cc22128d4d6bb25de2f5acc085c4 2024-11-13T18:25:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/4d34beb8d3fd459e8b5f18af156d7f43 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/4d34beb8d3fd459e8b5f18af156d7f43 2024-11-13T18:25:54,607 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/4d34beb8d3fd459e8b5f18af156d7f43, entries=150, sequenceid=167, filesize=11.9 K 2024-11-13T18:25:54,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/644f6bdc8aef4e41b0e21fd0ca4fa437 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/644f6bdc8aef4e41b0e21fd0ca4fa437 2024-11-13T18:25:54,613 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/644f6bdc8aef4e41b0e21fd0ca4fa437, entries=150, sequenceid=167, filesize=11.9 K 2024-11-13T18:25:54,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/0031cc22128d4d6bb25de2f5acc085c4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/0031cc22128d4d6bb25de2f5acc085c4 2024-11-13T18:25:54,619 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/0031cc22128d4d6bb25de2f5acc085c4, entries=150, sequenceid=167, filesize=11.9 K 2024-11-13T18:25:54,621 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for ecb8a98d59975d334af6c23fb4444cd4 in 1155ms, sequenceid=167, compaction requested=false 2024-11-13T18:25:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-13T18:25:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-13T18:25:54,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-13T18:25:54,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6680 sec 2024-11-13T18:25:54,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.7000 sec 2024-11-13T18:25:55,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:25:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:55,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/7209d9393c674dce9e2143d71b065f88 is 50, key is test_row_0/A:col10/1731522355028/Put/seqid=0 2024-11-13T18:25:55,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742297_1473 (size=12151) 2024-11-13T18:25:55,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/7209d9393c674dce9e2143d71b065f88 2024-11-13T18:25:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T18:25:55,116 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:55,117 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:55,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/39170ff8ca514ba490ec1ca8480c2452 is 50, key is test_row_0/B:col10/1731522355028/Put/seqid=0 2024-11-13T18:25:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-13T18:25:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T18:25:55,122 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:55,122 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:55,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:55,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742298_1474 (size=12151) 2024-11-13T18:25:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T18:25:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522415207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522415208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:55,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522415224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:55,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:55,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T18:25:55,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:55,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:55,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:55,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:55,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:55,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522415346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:55,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522415346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:55,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:55,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522415356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T18:25:55,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T18:25:55,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:55,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:55,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:55,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:55,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:55,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/39170ff8ca514ba490ec1ca8480c2452 2024-11-13T18:25:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522415575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522415575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:55,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/bf9470bd1da24010906341e57f54201b is 50, key is test_row_0/C:col10/1731522355028/Put/seqid=0 2024-11-13T18:25:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522415586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:55,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T18:25:55,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:55,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:55,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:55,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:55,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:55,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742299_1475 (size=12151) 2024-11-13T18:25:55,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/bf9470bd1da24010906341e57f54201b 2024-11-13T18:25:55,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/7209d9393c674dce9e2143d71b065f88 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/7209d9393c674dce9e2143d71b065f88 2024-11-13T18:25:55,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/7209d9393c674dce9e2143d71b065f88, entries=150, sequenceid=182, filesize=11.9 K 2024-11-13T18:25:55,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/39170ff8ca514ba490ec1ca8480c2452 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/39170ff8ca514ba490ec1ca8480c2452 2024-11-13T18:25:55,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/39170ff8ca514ba490ec1ca8480c2452, entries=150, sequenceid=182, filesize=11.9 K 2024-11-13T18:25:55,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/bf9470bd1da24010906341e57f54201b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/bf9470bd1da24010906341e57f54201b 2024-11-13T18:25:55,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/bf9470bd1da24010906341e57f54201b, entries=150, sequenceid=182, filesize=11.9 K 2024-11-13T18:25:55,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ecb8a98d59975d334af6c23fb4444cd4 in 654ms, sequenceid=182, compaction requested=true 2024-11-13T18:25:55,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:55,684 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:55,685 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:55,685 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/A is initiating minor compaction (all files) 2024-11-13T18:25:55,685 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/A in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:55,685 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/ba8b8eceab144060a4e4becfa84edc35, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/4d34beb8d3fd459e8b5f18af156d7f43, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/7209d9393c674dce9e2143d71b065f88] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=35.6 K 2024-11-13T18:25:55,686 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba8b8eceab144060a4e4becfa84edc35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731522352496 2024-11-13T18:25:55,686 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d34beb8d3fd459e8b5f18af156d7f43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731522352906 2024-11-13T18:25:55,687 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7209d9393c674dce9e2143d71b065f88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731522353747 2024-11-13T18:25:55,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:55,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:55,693 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:55,694 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:55,694 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/B is initiating minor compaction (all files) 2024-11-13T18:25:55,694 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/B in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:55,694 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/69b074555d5e4514861382bf9a6890d1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/644f6bdc8aef4e41b0e21fd0ca4fa437, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/39170ff8ca514ba490ec1ca8480c2452] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=35.6 K 2024-11-13T18:25:55,695 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b074555d5e4514861382bf9a6890d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731522352496 2024-11-13T18:25:55,695 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 644f6bdc8aef4e41b0e21fd0ca4fa437, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731522352906 2024-11-13T18:25:55,695 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 39170ff8ca514ba490ec1ca8480c2452, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731522353747 2024-11-13T18:25:55,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:55,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:55,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:55,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:55,703 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#A#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:55,704 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/5bf6c684f919418f8bd2fc62527a6468 is 50, key is test_row_0/A:col10/1731522355028/Put/seqid=0 2024-11-13T18:25:55,723 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#B#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:55,724 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/74443ba015dd404485a3bc12f0b842db is 50, key is test_row_0/B:col10/1731522355028/Put/seqid=0 2024-11-13T18:25:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T18:25:55,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T18:25:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:55,755 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:25:55,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:55,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:55,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:55,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:55,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:55,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:55,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742301_1477 (size=12391) 2024-11-13T18:25:55,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742300_1476 (size=12391) 2024-11-13T18:25:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/8b541161fe7b4297b247890ac31fe212 is 50, key is test_row_0/A:col10/1731522355201/Put/seqid=0 2024-11-13T18:25:55,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742302_1478 (size=12151) 2024-11-13T18:25:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:55,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:56,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522415984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522415990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522415994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522416118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522416119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522416119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,243 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/74443ba015dd404485a3bc12f0b842db as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/74443ba015dd404485a3bc12f0b842db 2024-11-13T18:25:56,250 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/5bf6c684f919418f8bd2fc62527a6468 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/5bf6c684f919418f8bd2fc62527a6468 2024-11-13T18:25:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T18:25:56,257 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/B of ecb8a98d59975d334af6c23fb4444cd4 into 74443ba015dd404485a3bc12f0b842db(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:56,257 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:56,257 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/B, priority=13, startTime=1731522355693; duration=0sec 2024-11-13T18:25:56,258 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:56,258 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:B 2024-11-13T18:25:56,258 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:56,263 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/A of ecb8a98d59975d334af6c23fb4444cd4 into 5bf6c684f919418f8bd2fc62527a6468(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:56,263 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:56,263 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/A, priority=13, startTime=1731522355684; duration=0sec 2024-11-13T18:25:56,263 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:56,263 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:A 2024-11-13T18:25:56,270 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/8b541161fe7b4297b247890ac31fe212 2024-11-13T18:25:56,270 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:56,270 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/C is initiating minor compaction (all files) 2024-11-13T18:25:56,270 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/C in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:56,270 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/402947b313184853b682a2cd2552ed37, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/0031cc22128d4d6bb25de2f5acc085c4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/bf9470bd1da24010906341e57f54201b] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=35.6 K 2024-11-13T18:25:56,276 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 402947b313184853b682a2cd2552ed37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731522352496 2024-11-13T18:25:56,278 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 0031cc22128d4d6bb25de2f5acc085c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731522352906 2024-11-13T18:25:56,281 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting bf9470bd1da24010906341e57f54201b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731522353747 2024-11-13T18:25:56,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/d3d759b574284391adbbeaa4f9fd87da is 50, key is test_row_0/B:col10/1731522355201/Put/seqid=0 2024-11-13T18:25:56,338 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#C#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:56,339 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/e63a75c39be4423a8f05539f1b248b88 is 50, key is test_row_0/C:col10/1731522355028/Put/seqid=0 2024-11-13T18:25:56,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742303_1479 (size=12151) 2024-11-13T18:25:56,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522416345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522416345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522416359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742304_1480 (size=12391) 2024-11-13T18:25:56,400 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/e63a75c39be4423a8f05539f1b248b88 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e63a75c39be4423a8f05539f1b248b88 2024-11-13T18:25:56,419 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/C of ecb8a98d59975d334af6c23fb4444cd4 into e63a75c39be4423a8f05539f1b248b88(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:56,420 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:56,420 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/C, priority=13, startTime=1731522355701; duration=0sec 2024-11-13T18:25:56,420 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:56,420 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:C 2024-11-13T18:25:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522416666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522416667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522416696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:56,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:56,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:56,755 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/d3d759b574284391adbbeaa4f9fd87da 2024-11-13T18:25:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/7540bd9559434dbea5b1b8495e6dbe5b is 50, key is test_row_0/C:col10/1731522355201/Put/seqid=0 2024-11-13T18:25:56,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742305_1481 (size=12151) 2024-11-13T18:25:56,820 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/7540bd9559434dbea5b1b8495e6dbe5b 2024-11-13T18:25:56,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/8b541161fe7b4297b247890ac31fe212 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/8b541161fe7b4297b247890ac31fe212 2024-11-13T18:25:56,836 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/8b541161fe7b4297b247890ac31fe212, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T18:25:56,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/d3d759b574284391adbbeaa4f9fd87da as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d3d759b574284391adbbeaa4f9fd87da 2024-11-13T18:25:56,853 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d3d759b574284391adbbeaa4f9fd87da, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T18:25:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/7540bd9559434dbea5b1b8495e6dbe5b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7540bd9559434dbea5b1b8495e6dbe5b 2024-11-13T18:25:56,871 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7540bd9559434dbea5b1b8495e6dbe5b, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T18:25:56,872 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for ecb8a98d59975d334af6c23fb4444cd4 in 1117ms, sequenceid=202, compaction requested=false 2024-11-13T18:25:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-13T18:25:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-13T18:25:56,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-13T18:25:56,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7530 sec 2024-11-13T18:25:56,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.7610 sec 2024-11-13T18:25:57,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T18:25:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:57,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/b063d0f3fe964d3db9fb0ccb2d320c52 is 50, key is test_row_0/A:col10/1731522355984/Put/seqid=0 2024-11-13T18:25:57,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742306_1482 (size=14541) 2024-11-13T18:25:57,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/b063d0f3fe964d3db9fb0ccb2d320c52 2024-11-13T18:25:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T18:25:57,268 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:25:57,268 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:25:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-13T18:25:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T18:25:57,274 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:25:57,278 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:25:57,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:25:57,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/f8db36d6793a4ceb85d3e53179b1a9b3 is 50, key is test_row_0/B:col10/1731522355984/Put/seqid=0 2024-11-13T18:25:57,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522417295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:57,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522417299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:57,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522417303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:57,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742307_1483 (size=12151) 2024-11-13T18:25:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T18:25:57,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:57,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:57,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:57,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:57,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522417426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:57,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522417426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:57,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522417436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:57,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:57,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:57,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T18:25:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522417645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:57,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522417645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:57,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:57,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522417685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:57,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:57,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:57,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:57,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:57,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/f8db36d6793a4ceb85d3e53179b1a9b3 2024-11-13T18:25:57,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/b28dfc2530ff4d14a799bd1d12c3f80a is 50, key is test_row_0/C:col10/1731522355984/Put/seqid=0 2024-11-13T18:25:57,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742308_1484 (size=12151) 2024-11-13T18:25:57,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/b28dfc2530ff4d14a799bd1d12c3f80a 2024-11-13T18:25:57,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/b063d0f3fe964d3db9fb0ccb2d320c52 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b063d0f3fe964d3db9fb0ccb2d320c52 2024-11-13T18:25:57,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b063d0f3fe964d3db9fb0ccb2d320c52, entries=200, sequenceid=219, filesize=14.2 K 2024-11-13T18:25:57,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/f8db36d6793a4ceb85d3e53179b1a9b3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f8db36d6793a4ceb85d3e53179b1a9b3 2024-11-13T18:25:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T18:25:57,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:57,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:57,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:57,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:57,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f8db36d6793a4ceb85d3e53179b1a9b3, entries=150, sequenceid=219, filesize=11.9 K 2024-11-13T18:25:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-13T18:25:57,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/b28dfc2530ff4d14a799bd1d12c3f80a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/b28dfc2530ff4d14a799bd1d12c3f80a 2024-11-13T18:25:57,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/b28dfc2530ff4d14a799bd1d12c3f80a, entries=150, sequenceid=219, filesize=11.9 K 2024-11-13T18:25:57,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ecb8a98d59975d334af6c23fb4444cd4 in 746ms, sequenceid=219, compaction requested=true 2024-11-13T18:25:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:25:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:25:57,946 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:57,946 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:57,948 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:57,948 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/C is initiating minor compaction (all files) 2024-11-13T18:25:57,948 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/C in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:57,949 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e63a75c39be4423a8f05539f1b248b88, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7540bd9559434dbea5b1b8495e6dbe5b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/b28dfc2530ff4d14a799bd1d12c3f80a] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=35.8 K 2024-11-13T18:25:57,949 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:57,949 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/A is initiating minor compaction (all files) 2024-11-13T18:25:57,949 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/A in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:57,949 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/5bf6c684f919418f8bd2fc62527a6468, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/8b541161fe7b4297b247890ac31fe212, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b063d0f3fe964d3db9fb0ccb2d320c52] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=38.2 K 2024-11-13T18:25:57,949 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e63a75c39be4423a8f05539f1b248b88, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731522353747 2024-11-13T18:25:57,949 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bf6c684f919418f8bd2fc62527a6468, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731522353747 2024-11-13T18:25:57,950 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 7540bd9559434dbea5b1b8495e6dbe5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522355195 2024-11-13T18:25:57,950 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b541161fe7b4297b247890ac31fe212, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522355195 2024-11-13T18:25:57,950 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting b28dfc2530ff4d14a799bd1d12c3f80a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731522355971 2024-11-13T18:25:57,950 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b063d0f3fe964d3db9fb0ccb2d320c52, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731522355971 2024-11-13T18:25:57,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:57,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:25:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:57,982 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:57,983 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/6ed7e8bdc3534656930dca7cfc1f78f6 is 50, key is test_row_0/A:col10/1731522355984/Put/seqid=0 2024-11-13T18:25:58,000 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#C#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:58,001 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/c3a65affe4874016a0d43008f4ea8d17 is 50, key is test_row_0/C:col10/1731522355984/Put/seqid=0 2024-11-13T18:25:58,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/29b7ff6c6b524560941ed97ef1ec32cc is 50, key is test_row_0/A:col10/1731522357962/Put/seqid=0 2024-11-13T18:25:58,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522418024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522418024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:58,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:58,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522418037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522418053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522418064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742309_1485 (size=12493) 2024-11-13T18:25:58,108 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/6ed7e8bdc3534656930dca7cfc1f78f6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/6ed7e8bdc3534656930dca7cfc1f78f6 2024-11-13T18:25:58,125 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/A of ecb8a98d59975d334af6c23fb4444cd4 into 6ed7e8bdc3534656930dca7cfc1f78f6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:58,125 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:58,125 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/A, priority=13, startTime=1731522357946; duration=0sec 2024-11-13T18:25:58,125 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:58,125 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:A 2024-11-13T18:25:58,126 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:58,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742311_1487 (size=14541) 2024-11-13T18:25:58,132 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:58,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/29b7ff6c6b524560941ed97ef1ec32cc 2024-11-13T18:25:58,133 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/B is initiating minor compaction (all files) 2024-11-13T18:25:58,133 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/B in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,133 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/74443ba015dd404485a3bc12f0b842db, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d3d759b574284391adbbeaa4f9fd87da, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f8db36d6793a4ceb85d3e53179b1a9b3] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=35.8 K 2024-11-13T18:25:58,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742310_1486 (size=12493) 2024-11-13T18:25:58,137 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74443ba015dd404485a3bc12f0b842db, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731522353747 2024-11-13T18:25:58,139 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3d759b574284391adbbeaa4f9fd87da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522355195 2024-11-13T18:25:58,140 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8db36d6793a4ceb85d3e53179b1a9b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731522355971 2024-11-13T18:25:58,145 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/c3a65affe4874016a0d43008f4ea8d17 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/c3a65affe4874016a0d43008f4ea8d17 2024-11-13T18:25:58,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/c3d9a0b69fac4b89911828814e67598c is 50, key is test_row_0/B:col10/1731522357962/Put/seqid=0 2024-11-13T18:25:58,162 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/C of ecb8a98d59975d334af6c23fb4444cd4 into c3a65affe4874016a0d43008f4ea8d17(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:58,162 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:58,162 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/C, priority=13, startTime=1731522357946; duration=0sec 2024-11-13T18:25:58,162 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:58,162 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:C 2024-11-13T18:25:58,173 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#B#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:58,174 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/c3b249cf47bf4f4d8dc736cd77dbefac is 50, key is test_row_0/B:col10/1731522355984/Put/seqid=0 2024-11-13T18:25:58,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522418165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522418166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522418176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522418177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522418193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:58,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:58,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742313_1489 (size=12493) 2024-11-13T18:25:58,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742312_1488 (size=12151) 2024-11-13T18:25:58,244 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/c3b249cf47bf4f4d8dc736cd77dbefac as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3b249cf47bf4f4d8dc736cd77dbefac 2024-11-13T18:25:58,252 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/B of ecb8a98d59975d334af6c23fb4444cd4 into c3b249cf47bf4f4d8dc736cd77dbefac(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:58,252 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:58,252 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/B, priority=13, startTime=1731522357946; duration=0sec 2024-11-13T18:25:58,252 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:58,252 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:B 2024-11-13T18:25:58,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:58,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:58,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T18:25:58,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522418399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522418403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522418404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522418406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522418416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:58,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:58,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/c3d9a0b69fac4b89911828814e67598c 2024-11-13T18:25:58,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/e30fb7beef5a43a896a6d22fd16fcc8e is 50, key is test_row_0/C:col10/1731522357962/Put/seqid=0 2024-11-13T18:25:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522418726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522418726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522418726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522418738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522418745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:58,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742314_1490 (size=12151) 2024-11-13T18:25:58,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:58,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:58,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:58,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:59,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:59,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/e30fb7beef5a43a896a6d22fd16fcc8e 2024-11-13T18:25:59,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/29b7ff6c6b524560941ed97ef1ec32cc as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/29b7ff6c6b524560941ed97ef1ec32cc 2024-11-13T18:25:59,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/29b7ff6c6b524560941ed97ef1ec32cc, entries=200, sequenceid=238, filesize=14.2 K 2024-11-13T18:25:59,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/c3d9a0b69fac4b89911828814e67598c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3d9a0b69fac4b89911828814e67598c 2024-11-13T18:25:59,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3d9a0b69fac4b89911828814e67598c, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T18:25:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/e30fb7beef5a43a896a6d22fd16fcc8e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e30fb7beef5a43a896a6d22fd16fcc8e 2024-11-13T18:25:59,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e30fb7beef5a43a896a6d22fd16fcc8e, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T18:25:59,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for ecb8a98d59975d334af6c23fb4444cd4 in 1245ms, sequenceid=238, compaction requested=false 2024-11-13T18:25:59,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:59,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:25:59,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:59,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:59,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:59,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:59,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:59,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:59,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/278ff840142941f9b462a7059c6996a3 is 50, key is test_row_0/A:col10/1731522358022/Put/seqid=0 2024-11-13T18:25:59,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742315_1491 (size=14541) 2024-11-13T18:25:59,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/278ff840142941f9b462a7059c6996a3 2024-11-13T18:25:59,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522419292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522419307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522419308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522419310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522419311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/5befdb024c134e248580805bafa2be69 is 50, key is test_row_0/B:col10/1731522358022/Put/seqid=0 2024-11-13T18:25:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:59,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:59,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742316_1492 (size=12151) 2024-11-13T18:25:59,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/5befdb024c134e248580805bafa2be69 2024-11-13T18:25:59,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/777edf2c361a4566bbce8f640ad5d1d8 is 50, key is test_row_0/C:col10/1731522358022/Put/seqid=0 2024-11-13T18:25:59,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742317_1493 (size=12151) 2024-11-13T18:25:59,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/777edf2c361a4566bbce8f640ad5d1d8 2024-11-13T18:25:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T18:25:59,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522419420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522419419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/278ff840142941f9b462a7059c6996a3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/278ff840142941f9b462a7059c6996a3 2024-11-13T18:25:59,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522419436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522419436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522419439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/278ff840142941f9b462a7059c6996a3, entries=200, sequenceid=256, filesize=14.2 K 2024-11-13T18:25:59,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/5befdb024c134e248580805bafa2be69 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5befdb024c134e248580805bafa2be69 2024-11-13T18:25:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5befdb024c134e248580805bafa2be69, entries=150, sequenceid=256, filesize=11.9 K 2024-11-13T18:25:59,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/777edf2c361a4566bbce8f640ad5d1d8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/777edf2c361a4566bbce8f640ad5d1d8 2024-11-13T18:25:59,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:59,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:59,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/777edf2c361a4566bbce8f640ad5d1d8, entries=150, sequenceid=256, filesize=11.9 K 2024-11-13T18:25:59,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ecb8a98d59975d334af6c23fb4444cd4 in 229ms, sequenceid=256, compaction requested=true 2024-11-13T18:25:59,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:59,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:25:59,492 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:59,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:59,492 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:59,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:25:59,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:25:59,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:25:59,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:59,493 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:59,494 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/A is initiating minor compaction (all files) 2024-11-13T18:25:59,494 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:59,494 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/A in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,494 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/B is initiating minor compaction (all files) 2024-11-13T18:25:59,494 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/6ed7e8bdc3534656930dca7cfc1f78f6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/29b7ff6c6b524560941ed97ef1ec32cc, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/278ff840142941f9b462a7059c6996a3] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=40.6 K 2024-11-13T18:25:59,494 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/B in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,494 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3b249cf47bf4f4d8dc736cd77dbefac, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3d9a0b69fac4b89911828814e67598c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5befdb024c134e248580805bafa2be69] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=35.9 K 2024-11-13T18:25:59,494 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ed7e8bdc3534656930dca7cfc1f78f6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731522355971 2024-11-13T18:25:59,494 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b249cf47bf4f4d8dc736cd77dbefac, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731522355971 2024-11-13T18:25:59,494 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b7ff6c6b524560941ed97ef1ec32cc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731522357268 2024-11-13T18:25:59,495 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c3d9a0b69fac4b89911828814e67598c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731522357297 2024-11-13T18:25:59,495 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 278ff840142941f9b462a7059c6996a3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731522358022 2024-11-13T18:25:59,495 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 5befdb024c134e248580805bafa2be69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731522358022 2024-11-13T18:25:59,528 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:59,529 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/2c27cc8e6e01473ea0b5527adc0315c3 is 50, key is test_row_0/A:col10/1731522358022/Put/seqid=0 2024-11-13T18:25:59,561 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:59,562 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/bac7027d9f2a48e183b67a86323dcb88 is 50, key is test_row_0/B:col10/1731522358022/Put/seqid=0 2024-11-13T18:25:59,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742318_1494 (size=12595) 2024-11-13T18:25:59,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742319_1495 (size=12595) 2024-11-13T18:25:59,612 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/bac7027d9f2a48e183b67a86323dcb88 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/bac7027d9f2a48e183b67a86323dcb88 2024-11-13T18:25:59,618 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/B of ecb8a98d59975d334af6c23fb4444cd4 into bac7027d9f2a48e183b67a86323dcb88(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:59,618 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:25:59,618 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/B, priority=13, startTime=1731522359492; duration=0sec 2024-11-13T18:25:59,618 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:25:59,618 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:B 2024-11-13T18:25:59,618 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:25:59,621 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:25:59,621 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/C is initiating minor compaction (all files) 2024-11-13T18:25:59,621 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/C in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,621 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/c3a65affe4874016a0d43008f4ea8d17, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e30fb7beef5a43a896a6d22fd16fcc8e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/777edf2c361a4566bbce8f640ad5d1d8] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=35.9 K 2024-11-13T18:25:59,622 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c3a65affe4874016a0d43008f4ea8d17, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731522355971 2024-11-13T18:25:59,622 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e30fb7beef5a43a896a6d22fd16fcc8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731522357297 2024-11-13T18:25:59,623 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 777edf2c361a4566bbce8f640ad5d1d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731522358022 2024-11-13T18:25:59,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T18:25:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:25:59,640 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:25:59,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:25:59,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:59,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:25:59,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:59,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:25:59,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:25:59,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:25:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:25:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/b8cab50f3bae42db97fe7199b8fbe2a2 is 50, key is test_row_0/A:col10/1731522359307/Put/seqid=0 2024-11-13T18:25:59,669 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#C#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:59,669 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/7d25205631d141cea564705776b49e34 is 50, key is test_row_0/C:col10/1731522358022/Put/seqid=0 2024-11-13T18:25:59,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742320_1496 (size=12301) 2024-11-13T18:25:59,700 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/b8cab50f3bae42db97fe7199b8fbe2a2 2024-11-13T18:25:59,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522419692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/0349d23281ee4b3297c22f91db2b95ad is 50, key is test_row_0/B:col10/1731522359307/Put/seqid=0 2024-11-13T18:25:59,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742321_1497 (size=12595) 2024-11-13T18:25:59,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522419695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522419697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522419701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522419703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742322_1498 (size=12301) 2024-11-13T18:25:59,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522419816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522419825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522419826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522419827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:25:59,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:25:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522419839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:25:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:25:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,030 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/2c27cc8e6e01473ea0b5527adc0315c3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2c27cc8e6e01473ea0b5527adc0315c3 2024-11-13T18:26:00,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522420046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522420046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,062 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/A of ecb8a98d59975d334af6c23fb4444cd4 into 2c27cc8e6e01473ea0b5527adc0315c3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:00,062 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:00,062 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/A, priority=13, startTime=1731522359492; duration=0sec 2024-11-13T18:26:00,063 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:00,063 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:A 2024-11-13T18:26:00,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522420050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522420055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522420058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,139 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/7d25205631d141cea564705776b49e34 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7d25205631d141cea564705776b49e34 2024-11-13T18:26:00,148 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/0349d23281ee4b3297c22f91db2b95ad 2024-11-13T18:26:00,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/9d2c5a3d9ea2438aa8ab101051b94746 is 50, key is test_row_0/C:col10/1731522359307/Put/seqid=0 2024-11-13T18:26:00,178 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/C of ecb8a98d59975d334af6c23fb4444cd4 into 7d25205631d141cea564705776b49e34(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:00,178 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:00,178 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/C, priority=13, startTime=1731522359493; duration=0sec 2024-11-13T18:26:00,178 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:00,178 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:C 2024-11-13T18:26:00,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742323_1499 (size=12301) 2024-11-13T18:26:00,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522420366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522420367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522420375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522420379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522420388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,613 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/9d2c5a3d9ea2438aa8ab101051b94746 2024-11-13T18:26:00,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/b8cab50f3bae42db97fe7199b8fbe2a2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b8cab50f3bae42db97fe7199b8fbe2a2 2024-11-13T18:26:00,630 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b8cab50f3bae42db97fe7199b8fbe2a2, entries=150, sequenceid=274, filesize=12.0 K 2024-11-13T18:26:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/0349d23281ee4b3297c22f91db2b95ad as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0349d23281ee4b3297c22f91db2b95ad 2024-11-13T18:26:00,637 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0349d23281ee4b3297c22f91db2b95ad, entries=150, sequenceid=274, filesize=12.0 K 2024-11-13T18:26:00,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/9d2c5a3d9ea2438aa8ab101051b94746 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/9d2c5a3d9ea2438aa8ab101051b94746 2024-11-13T18:26:00,656 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/9d2c5a3d9ea2438aa8ab101051b94746, entries=150, sequenceid=274, filesize=12.0 K 2024-11-13T18:26:00,658 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ecb8a98d59975d334af6c23fb4444cd4 in 1019ms, sequenceid=274, compaction requested=false 2024-11-13T18:26:00,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:00,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:00,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-13T18:26:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-13T18:26:00,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-13T18:26:00,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3820 sec 2024-11-13T18:26:00,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 3.3960 sec 2024-11-13T18:26:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:00,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:26:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:26:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:26:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:26:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:00,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/caca4492c76c4898b8abeb9b62f32ec9 is 50, key is test_row_0/A:col10/1731522359701/Put/seqid=0 2024-11-13T18:26:00,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742324_1500 (size=14741) 2024-11-13T18:26:00,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522420915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522420932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522420932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522420935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522420935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522421045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522421056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522421056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522421058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522421069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522421256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522421275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522421277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522421277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522421286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/caca4492c76c4898b8abeb9b62f32ec9 2024-11-13T18:26:01,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/b4c7b72a7213430381675a500f7e7018 is 50, key is test_row_0/B:col10/1731522359701/Put/seqid=0 2024-11-13T18:26:01,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742325_1501 (size=12301) 2024-11-13T18:26:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T18:26:01,437 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:01,437 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:26:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-13T18:26:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:26:01,441 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:26:01,442 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:26:01,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:26:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:26:01,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522421566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522421586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522421586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522421599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522421600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:01,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:26:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:01,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:26:01,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:26:01,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:01,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:01,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:01,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:01,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:01,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/b4c7b72a7213430381675a500f7e7018 2024-11-13T18:26:01,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/5abeb50403644555b172eef7b19c68aa is 50, key is test_row_0/C:col10/1731522359701/Put/seqid=0 2024-11-13T18:26:01,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742326_1502 (size=12301) 2024-11-13T18:26:01,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/5abeb50403644555b172eef7b19c68aa 2024-11-13T18:26:01,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/caca4492c76c4898b8abeb9b62f32ec9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/caca4492c76c4898b8abeb9b62f32ec9 2024-11-13T18:26:01,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:26:01,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:01,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:01,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:01,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:01,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:01,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/caca4492c76c4898b8abeb9b62f32ec9, entries=200, sequenceid=292, filesize=14.4 K 2024-11-13T18:26:01,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/b4c7b72a7213430381675a500f7e7018 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b4c7b72a7213430381675a500f7e7018 2024-11-13T18:26:02,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b4c7b72a7213430381675a500f7e7018, entries=150, sequenceid=292, filesize=12.0 K 2024-11-13T18:26:02,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/5abeb50403644555b172eef7b19c68aa as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abeb50403644555b172eef7b19c68aa 2024-11-13T18:26:02,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abeb50403644555b172eef7b19c68aa, entries=150, sequenceid=292, filesize=12.0 K 2024-11-13T18:26:02,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ecb8a98d59975d334af6c23fb4444cd4 in 1153ms, sequenceid=292, compaction requested=true 2024-11-13T18:26:02,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:02,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:02,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:02,043 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:02,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:02,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:02,044 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:02,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:02,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:02,063 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:02,063 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/A is initiating minor compaction (all files) 2024-11-13T18:26:02,063 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/A in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,063 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2c27cc8e6e01473ea0b5527adc0315c3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b8cab50f3bae42db97fe7199b8fbe2a2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/caca4492c76c4898b8abeb9b62f32ec9] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=38.7 K 2024-11-13T18:26:02,066 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:02,067 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/B is initiating minor compaction (all files) 2024-11-13T18:26:02,067 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/B in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,067 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/bac7027d9f2a48e183b67a86323dcb88, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0349d23281ee4b3297c22f91db2b95ad, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b4c7b72a7213430381675a500f7e7018] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=36.3 K 2024-11-13T18:26:02,068 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c27cc8e6e01473ea0b5527adc0315c3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731522358022 2024-11-13T18:26:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:26:02,071 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting b8cab50f3bae42db97fe7199b8fbe2a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731522359305 2024-11-13T18:26:02,075 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting bac7027d9f2a48e183b67a86323dcb88, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731522358022 2024-11-13T18:26:02,075 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting caca4492c76c4898b8abeb9b62f32ec9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731522359669 2024-11-13T18:26:02,083 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0349d23281ee4b3297c22f91db2b95ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731522359305 2024-11-13T18:26:02,088 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c7b72a7213430381675a500f7e7018, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731522359699 2024-11-13T18:26:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:02,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:26:02,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:26:02,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:02,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:26:02,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:02,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:26:02,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:02,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:26:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/532484058dc2449884c72cd1f54fef76 is 50, key is test_row_0/A:col10/1731522362095/Put/seqid=0 2024-11-13T18:26:02,145 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#B#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:02,145 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#A#compaction#427 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:02,145 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/0dc95a534a454f48ad0beff6679cf6ed is 50, key is test_row_0/B:col10/1731522359701/Put/seqid=0 2024-11-13T18:26:02,146 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/68e36445aa3e41c5807e92f57605972b is 50, key is test_row_0/A:col10/1731522359701/Put/seqid=0 2024-11-13T18:26:02,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522422148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522422155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522422155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742327_1503 (size=14741) 2024-11-13T18:26:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522422172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522422183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/532484058dc2449884c72cd1f54fef76 2024-11-13T18:26:02,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742328_1504 (size=12847) 2024-11-13T18:26:02,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742329_1505 (size=12847) 2024-11-13T18:26:02,233 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/68e36445aa3e41c5807e92f57605972b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/68e36445aa3e41c5807e92f57605972b 2024-11-13T18:26:02,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/8b99627fc3fb4c7d8951a5a3d85ec7f9 is 50, key is test_row_0/B:col10/1731522362095/Put/seqid=0 2024-11-13T18:26:02,263 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/A of ecb8a98d59975d334af6c23fb4444cd4 into 68e36445aa3e41c5807e92f57605972b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:02,263 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:02,263 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/A, priority=13, startTime=1731522362043; duration=0sec 2024-11-13T18:26:02,263 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:02,263 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:A 2024-11-13T18:26:02,263 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:02,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742330_1506 (size=12301) 2024-11-13T18:26:02,275 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:02,275 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/C is initiating minor compaction (all files) 2024-11-13T18:26:02,275 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/C in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,275 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7d25205631d141cea564705776b49e34, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/9d2c5a3d9ea2438aa8ab101051b94746, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abeb50403644555b172eef7b19c68aa] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=36.3 K 2024-11-13T18:26:02,278 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d25205631d141cea564705776b49e34, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731522358022 2024-11-13T18:26:02,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:26:02,287 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d2c5a3d9ea2438aa8ab101051b94746, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731522359305 2024-11-13T18:26:02,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:02,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,290 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 5abeb50403644555b172eef7b19c68aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731522359699 2024-11-13T18:26:02,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522422289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522422302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522422302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522422306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522422306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,357 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:02,357 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/e688e484c343475884e88dab8af5d290 is 50, key is test_row_0/C:col10/1731522359701/Put/seqid=0 2024-11-13T18:26:02,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742331_1507 (size=12847) 2024-11-13T18:26:02,436 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/e688e484c343475884e88dab8af5d290 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e688e484c343475884e88dab8af5d290 2024-11-13T18:26:02,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:26:02,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:02,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,448 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/C of ecb8a98d59975d334af6c23fb4444cd4 into e688e484c343475884e88dab8af5d290(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:02,448 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:02,448 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/C, priority=13, startTime=1731522362047; duration=0sec 2024-11-13T18:26:02,448 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:02,448 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:C 2024-11-13T18:26:02,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522422506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522422525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522422525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522422526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522422526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:26:02,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:26:02,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:02,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:02,624 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/0dc95a534a454f48ad0beff6679cf6ed as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0dc95a534a454f48ad0beff6679cf6ed 2024-11-13T18:26:02,629 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/B of ecb8a98d59975d334af6c23fb4444cd4 into 0dc95a534a454f48ad0beff6679cf6ed(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:02,629 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:02,629 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/B, priority=13, startTime=1731522362043; duration=0sec 2024-11-13T18:26:02,629 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:02,629 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:B 2024-11-13T18:26:02,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/8b99627fc3fb4c7d8951a5a3d85ec7f9 2024-11-13T18:26:02,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/2e9c313bd9e54dea8ff31507487ae229 is 50, key is test_row_0/C:col10/1731522362095/Put/seqid=0 2024-11-13T18:26:02,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742332_1508 (size=12301) 2024-11-13T18:26:02,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/2e9c313bd9e54dea8ff31507487ae229 2024-11-13T18:26:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/532484058dc2449884c72cd1f54fef76 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/532484058dc2449884c72cd1f54fef76 2024-11-13T18:26:02,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/532484058dc2449884c72cd1f54fef76, entries=200, sequenceid=309, filesize=14.4 K 2024-11-13T18:26:02,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/8b99627fc3fb4c7d8951a5a3d85ec7f9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8b99627fc3fb4c7d8951a5a3d85ec7f9 2024-11-13T18:26:02,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8b99627fc3fb4c7d8951a5a3d85ec7f9, entries=150, sequenceid=309, filesize=12.0 K 2024-11-13T18:26:02,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/2e9c313bd9e54dea8ff31507487ae229 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2e9c313bd9e54dea8ff31507487ae229 2024-11-13T18:26:02,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2e9c313bd9e54dea8ff31507487ae229, entries=150, sequenceid=309, filesize=12.0 K 2024-11-13T18:26:02,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ecb8a98d59975d334af6c23fb4444cd4 in 643ms, sequenceid=309, compaction requested=false 2024-11-13T18:26:02,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:02,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:26:02,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:02,757 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:26:02,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:26:02,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:02,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:26:02,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:02,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:26:02,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/17f1d8c9781b4a2d9fbb380cb89a2f48 is 50, key is test_row_0/A:col10/1731522362171/Put/seqid=0 2024-11-13T18:26:02,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742333_1509 (size=12301) 2024-11-13T18:26:02,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:02,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522422897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522422898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522422898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522422904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522422914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522423026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522423026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522423026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522423027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522423036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,207 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/17f1d8c9781b4a2d9fbb380cb89a2f48 2024-11-13T18:26:03,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/d2256f7bda8e44ae893a754e49178a32 is 50, key is test_row_0/B:col10/1731522362171/Put/seqid=0 2024-11-13T18:26:03,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522423245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522423246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522423246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522423248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522423259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742334_1510 (size=12301) 2024-11-13T18:26:03,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522423558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522423558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522423558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522423567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:26:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522423587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:03,720 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/d2256f7bda8e44ae893a754e49178a32 2024-11-13T18:26:03,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/5abac2b288e6440995bace61f4a151ee is 50, key is test_row_0/C:col10/1731522362171/Put/seqid=0 2024-11-13T18:26:03,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742335_1511 (size=12301) 2024-11-13T18:26:04,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522424098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:04,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522424099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:04,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522424102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:04,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:04,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522424106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:04,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:04,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:04,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522424118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:04,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:04,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:04,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:04,196 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/5abac2b288e6440995bace61f4a151ee 2024-11-13T18:26:04,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/17f1d8c9781b4a2d9fbb380cb89a2f48 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/17f1d8c9781b4a2d9fbb380cb89a2f48 2024-11-13T18:26:04,230 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/17f1d8c9781b4a2d9fbb380cb89a2f48, entries=150, sequenceid=327, filesize=12.0 K 2024-11-13T18:26:04,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/d2256f7bda8e44ae893a754e49178a32 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d2256f7bda8e44ae893a754e49178a32 2024-11-13T18:26:04,255 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d2256f7bda8e44ae893a754e49178a32, entries=150, sequenceid=327, filesize=12.0 K 2024-11-13T18:26:04,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/5abac2b288e6440995bace61f4a151ee as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abac2b288e6440995bace61f4a151ee 2024-11-13T18:26:04,271 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abac2b288e6440995bace61f4a151ee, entries=150, sequenceid=327, filesize=12.0 K 2024-11-13T18:26:04,273 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ecb8a98d59975d334af6c23fb4444cd4 in 1515ms, sequenceid=327, compaction requested=true 2024-11-13T18:26:04,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:04,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:04,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-13T18:26:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-13T18:26:04,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-13T18:26:04,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8450 sec 2024-11-13T18:26:04,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.8580 sec 2024-11-13T18:26:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:05,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T18:26:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:26:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:26:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:26:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:05,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/51763102abe74d2685b98145fe8693fb is 50, key is test_row_0/A:col10/1731522365141/Put/seqid=0 2024-11-13T18:26:05,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522425199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522425203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522425205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522425207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522425208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742336_1512 (size=17181) 2024-11-13T18:26:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/51763102abe74d2685b98145fe8693fb 2024-11-13T18:26:05,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/1b0e721896f742d297233ea9f124a9a9 is 50, key is test_row_0/B:col10/1731522365141/Put/seqid=0 2024-11-13T18:26:05,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522425326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522425339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522425339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522425347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522425347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742337_1513 (size=12301) 2024-11-13T18:26:05,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522425555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522425555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522425556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522425556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522425556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:26:05,611 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:05,614 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:26:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-13T18:26:05,620 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:26:05,621 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:26:05,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:26:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:26:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:26:05,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:26:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:05,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:05,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/1b0e721896f742d297233ea9f124a9a9 2024-11-13T18:26:05,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/8f5567b297294d669fa05ef9838a71f1 is 50, key is test_row_0/C:col10/1731522365141/Put/seqid=0 2024-11-13T18:26:05,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522425876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522425879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522425879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522425882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522425883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:05,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:26:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:05,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:05,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742338_1514 (size=12301) 2024-11-13T18:26:05,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/8f5567b297294d669fa05ef9838a71f1 2024-11-13T18:26:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:26:05,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/51763102abe74d2685b98145fe8693fb as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/51763102abe74d2685b98145fe8693fb 2024-11-13T18:26:05,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/51763102abe74d2685b98145fe8693fb, entries=250, sequenceid=346, filesize=16.8 K 2024-11-13T18:26:06,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/1b0e721896f742d297233ea9f124a9a9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1b0e721896f742d297233ea9f124a9a9 2024-11-13T18:26:06,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1b0e721896f742d297233ea9f124a9a9, entries=150, sequenceid=346, filesize=12.0 K 2024-11-13T18:26:06,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/8f5567b297294d669fa05ef9838a71f1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/8f5567b297294d669fa05ef9838a71f1 2024-11-13T18:26:06,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/8f5567b297294d669fa05ef9838a71f1, entries=150, sequenceid=346, filesize=12.0 K 2024-11-13T18:26:06,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ecb8a98d59975d334af6c23fb4444cd4 in 915ms, sequenceid=346, compaction requested=true 2024-11-13T18:26:06,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:06,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:06,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:06,057 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:26:06,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:06,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:06,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:06,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:26:06,057 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:26:06,069 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:26:06,069 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57070 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:26:06,069 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/B is initiating minor compaction (all files) 2024-11-13T18:26:06,069 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/A is initiating minor compaction (all files) 2024-11-13T18:26:06,069 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/A in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:06,069 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/B in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:06,069 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0dc95a534a454f48ad0beff6679cf6ed, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8b99627fc3fb4c7d8951a5a3d85ec7f9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d2256f7bda8e44ae893a754e49178a32, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1b0e721896f742d297233ea9f124a9a9] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=48.6 K 2024-11-13T18:26:06,069 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/68e36445aa3e41c5807e92f57605972b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/532484058dc2449884c72cd1f54fef76, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/17f1d8c9781b4a2d9fbb380cb89a2f48, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/51763102abe74d2685b98145fe8693fb] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=55.7 K 2024-11-13T18:26:06,071 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 68e36445aa3e41c5807e92f57605972b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731522359699 2024-11-13T18:26:06,071 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dc95a534a454f48ad0beff6679cf6ed, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731522359699 2024-11-13T18:26:06,071 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 532484058dc2449884c72cd1f54fef76, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731522360909 2024-11-13T18:26:06,071 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b99627fc3fb4c7d8951a5a3d85ec7f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731522360909 2024-11-13T18:26:06,071 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 17f1d8c9781b4a2d9fbb380cb89a2f48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731522362137 2024-11-13T18:26:06,071 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2256f7bda8e44ae893a754e49178a32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731522362137 2024-11-13T18:26:06,071 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 51763102abe74d2685b98145fe8693fb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731522362893 2024-11-13T18:26:06,072 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b0e721896f742d297233ea9f124a9a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731522362897 2024-11-13T18:26:06,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:26:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:06,095 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:26:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:26:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:26:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:26:06,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:06,101 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:06,102 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/5743710ade874b95af0783e9b750fa28 is 50, key is test_row_0/B:col10/1731522365141/Put/seqid=0 2024-11-13T18:26:06,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/833ff12cdd0e4003971b38af7040bab1 is 50, key is test_row_0/A:col10/1731522365202/Put/seqid=0 2024-11-13T18:26:06,150 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#A#compaction#440 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:06,150 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/9fc038f108a04df2b7feda358124468f is 50, key is test_row_0/A:col10/1731522365141/Put/seqid=0 2024-11-13T18:26:06,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742339_1515 (size=12983) 2024-11-13T18:26:06,213 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/5743710ade874b95af0783e9b750fa28 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5743710ade874b95af0783e9b750fa28 2024-11-13T18:26:06,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742341_1517 (size=12983) 2024-11-13T18:26:06,219 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/B of ecb8a98d59975d334af6c23fb4444cd4 into 5743710ade874b95af0783e9b750fa28(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:06,219 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:06,220 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/B, priority=12, startTime=1731522366057; duration=0sec 2024-11-13T18:26:06,220 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:06,220 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:B 2024-11-13T18:26:06,220 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:26:06,221 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:26:06,221 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/C is initiating minor compaction (all files) 2024-11-13T18:26:06,221 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/C in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:06,222 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e688e484c343475884e88dab8af5d290, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2e9c313bd9e54dea8ff31507487ae229, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abac2b288e6440995bace61f4a151ee, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/8f5567b297294d669fa05ef9838a71f1] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=48.6 K 2024-11-13T18:26:06,222 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting e688e484c343475884e88dab8af5d290, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731522359699 2024-11-13T18:26:06,224 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e9c313bd9e54dea8ff31507487ae229, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731522360909 2024-11-13T18:26:06,228 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5abac2b288e6440995bace61f4a151ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731522362137 2024-11-13T18:26:06,229 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f5567b297294d669fa05ef9838a71f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731522362897 2024-11-13T18:26:06,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742340_1516 (size=12301) 2024-11-13T18:26:06,251 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#C#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:06,251 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/5ab01c3aaace49578f2d1846864e338a is 50, key is test_row_0/C:col10/1731522365141/Put/seqid=0 2024-11-13T18:26:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:26:06,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742342_1518 (size=12983) 2024-11-13T18:26:06,304 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/5ab01c3aaace49578f2d1846864e338a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5ab01c3aaace49578f2d1846864e338a 2024-11-13T18:26:06,320 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/C of ecb8a98d59975d334af6c23fb4444cd4 into 5ab01c3aaace49578f2d1846864e338a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:06,320 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:06,320 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/C, priority=12, startTime=1731522366057; duration=0sec 2024-11-13T18:26:06,320 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:06,320 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:C 2024-11-13T18:26:06,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522426468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522426485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522426489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522426505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522426490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522426606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522426606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522426616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,631 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/9fc038f108a04df2b7feda358124468f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/9fc038f108a04df2b7feda358124468f 2024-11-13T18:26:06,637 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/833ff12cdd0e4003971b38af7040bab1 2024-11-13T18:26:06,652 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/A of ecb8a98d59975d334af6c23fb4444cd4 into 9fc038f108a04df2b7feda358124468f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:06,652 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:06,652 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/A, priority=12, startTime=1731522366057; duration=0sec 2024-11-13T18:26:06,652 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:06,652 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:A 2024-11-13T18:26:06,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/c04ca5067e17488b80acb3a9f5b01b84 is 50, key is test_row_0/B:col10/1731522365202/Put/seqid=0 2024-11-13T18:26:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522426635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522426636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742343_1519 (size=12301) 2024-11-13T18:26:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:26:06,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522426827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522426827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522426840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522426866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:06,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522426866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,113 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/c04ca5067e17488b80acb3a9f5b01b84 2024-11-13T18:26:07,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/238b5ebd71574b0d9254e3085ed05499 is 50, key is test_row_0/C:col10/1731522365202/Put/seqid=0 2024-11-13T18:26:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522427156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522427156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742344_1520 (size=12301) 2024-11-13T18:26:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,175 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/238b5ebd71574b0d9254e3085ed05499 2024-11-13T18:26:07,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/833ff12cdd0e4003971b38af7040bab1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/833ff12cdd0e4003971b38af7040bab1 2024-11-13T18:26:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522427166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,185 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/833ff12cdd0e4003971b38af7040bab1, entries=150, sequenceid=359, filesize=12.0 K 2024-11-13T18:26:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/c04ca5067e17488b80acb3a9f5b01b84 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c04ca5067e17488b80acb3a9f5b01b84 2024-11-13T18:26:07,195 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c04ca5067e17488b80acb3a9f5b01b84, entries=150, sequenceid=359, filesize=12.0 K 2024-11-13T18:26:07,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/238b5ebd71574b0d9254e3085ed05499 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/238b5ebd71574b0d9254e3085ed05499 2024-11-13T18:26:07,200 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/238b5ebd71574b0d9254e3085ed05499, entries=150, sequenceid=359, filesize=12.0 K 2024-11-13T18:26:07,201 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ecb8a98d59975d334af6c23fb4444cd4 in 1106ms, sequenceid=359, compaction requested=false 2024-11-13T18:26:07,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:07,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:07,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-13T18:26:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-13T18:26:07,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-13T18:26:07,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2024-11-13T18:26:07,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.5920 sec 2024-11-13T18:26:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:07,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:26:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:26:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:26:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:26:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:07,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/e02c465a98894296a3a0e79500008ae5 is 50, key is test_row_0/A:col10/1731522367226/Put/seqid=0 2024-11-13T18:26:07,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742345_1521 (size=14741) 2024-11-13T18:26:07,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/e02c465a98894296a3a0e79500008ae5 2024-11-13T18:26:07,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/1a4a7d06138747a9a68060c5295396e5 is 50, key is test_row_0/B:col10/1731522367226/Put/seqid=0 2024-11-13T18:26:07,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742346_1522 (size=12301) 2024-11-13T18:26:07,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/1a4a7d06138747a9a68060c5295396e5 2024-11-13T18:26:07,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/98dc195a24a1490daf5fc01dddf693f8 is 50, key is test_row_0/C:col10/1731522367226/Put/seqid=0 2024-11-13T18:26:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522427330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522427348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742347_1523 (size=12301) 2024-11-13T18:26:07,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522427476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522427476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522427686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522427687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522427699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522427700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522427705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:26:07,779 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:07,780 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:26:07,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-13T18:26:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:26:07,796 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:26:07,799 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:26:07,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:26:07,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/98dc195a24a1490daf5fc01dddf693f8 2024-11-13T18:26:07,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/e02c465a98894296a3a0e79500008ae5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/e02c465a98894296a3a0e79500008ae5 2024-11-13T18:26:07,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/e02c465a98894296a3a0e79500008ae5, entries=200, sequenceid=382, filesize=14.4 K 2024-11-13T18:26:07,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/1a4a7d06138747a9a68060c5295396e5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1a4a7d06138747a9a68060c5295396e5 2024-11-13T18:26:07,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1a4a7d06138747a9a68060c5295396e5, entries=150, sequenceid=382, filesize=12.0 K 2024-11-13T18:26:07,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/98dc195a24a1490daf5fc01dddf693f8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/98dc195a24a1490daf5fc01dddf693f8 2024-11-13T18:26:07,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/98dc195a24a1490daf5fc01dddf693f8, entries=150, sequenceid=382, filesize=12.0 K 2024-11-13T18:26:07,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ecb8a98d59975d334af6c23fb4444cd4 in 603ms, sequenceid=382, compaction requested=true 2024-11-13T18:26:07,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:07,832 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:07,834 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:07,834 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/A is initiating minor compaction (all files) 2024-11-13T18:26:07,834 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/A in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:07,834 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/9fc038f108a04df2b7feda358124468f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/833ff12cdd0e4003971b38af7040bab1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/e02c465a98894296a3a0e79500008ae5] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=39.1 K 2024-11-13T18:26:07,834 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fc038f108a04df2b7feda358124468f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731522362897 2024-11-13T18:26:07,835 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 833ff12cdd0e4003971b38af7040bab1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731522365196 2024-11-13T18:26:07,835 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting e02c465a98894296a3a0e79500008ae5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731522366466 2024-11-13T18:26:07,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:07,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:07,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:07,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:07,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:07,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:26:07,844 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:07,848 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#A#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:07,848 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/da51c119185e43d4be72d2040fb9e9e5 is 50, key is test_row_0/A:col10/1731522367226/Put/seqid=0 2024-11-13T18:26:07,851 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:07,851 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/B is initiating minor compaction (all files) 2024-11-13T18:26:07,851 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/B in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:07,851 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5743710ade874b95af0783e9b750fa28, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c04ca5067e17488b80acb3a9f5b01b84, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1a4a7d06138747a9a68060c5295396e5] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=36.7 K 2024-11-13T18:26:07,854 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 5743710ade874b95af0783e9b750fa28, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731522362897 2024-11-13T18:26:07,858 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c04ca5067e17488b80acb3a9f5b01b84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731522365196 2024-11-13T18:26:07,858 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a4a7d06138747a9a68060c5295396e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731522366466 2024-11-13T18:26:07,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742348_1524 (size=13085) 2024-11-13T18:26:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:26:07,931 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#B#compaction#448 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:07,931 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/e11b0f00bc8941c983163a11a2034657 is 50, key is test_row_0/B:col10/1731522367226/Put/seqid=0 2024-11-13T18:26:07,933 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/da51c119185e43d4be72d2040fb9e9e5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/da51c119185e43d4be72d2040fb9e9e5 2024-11-13T18:26:07,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:26:07,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:07,959 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:26:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:26:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:26:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:26:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:07,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742349_1525 (size=13085) 2024-11-13T18:26:07,966 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/A of ecb8a98d59975d334af6c23fb4444cd4 into da51c119185e43d4be72d2040fb9e9e5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:07,966 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:07,966 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/A, priority=13, startTime=1731522367832; duration=0sec 2024-11-13T18:26:07,966 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:07,966 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:A 2024-11-13T18:26:07,966 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:07,968 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:07,968 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/C is initiating minor compaction (all files) 2024-11-13T18:26:07,968 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/C in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:07,969 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5ab01c3aaace49578f2d1846864e338a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/238b5ebd71574b0d9254e3085ed05499, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/98dc195a24a1490daf5fc01dddf693f8] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=36.7 K 2024-11-13T18:26:07,976 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ab01c3aaace49578f2d1846864e338a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731522362897 2024-11-13T18:26:07,977 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 238b5ebd71574b0d9254e3085ed05499, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731522365196 2024-11-13T18:26:07,983 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98dc195a24a1490daf5fc01dddf693f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731522366466 2024-11-13T18:26:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/3feaab01e8954e31912d0c8034b7addf is 50, key is test_row_0/A:col10/1731522367326/Put/seqid=0 2024-11-13T18:26:07,989 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/e11b0f00bc8941c983163a11a2034657 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/e11b0f00bc8941c983163a11a2034657 2024-11-13T18:26:08,019 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/B of ecb8a98d59975d334af6c23fb4444cd4 into e11b0f00bc8941c983163a11a2034657(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:08,019 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:08,019 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/B, priority=13, startTime=1731522367840; duration=0sec 2024-11-13T18:26:08,020 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:08,020 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:B 2024-11-13T18:26:08,030 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#C#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:08,031 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/89ab7024f78a4b02a0195ab3657e23b4 is 50, key is test_row_0/C:col10/1731522367226/Put/seqid=0 2024-11-13T18:26:08,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742350_1526 (size=12301) 2024-11-13T18:26:08,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. as already flushing 2024-11-13T18:26:08,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:08,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742351_1527 (size=13085) 2024-11-13T18:26:08,094 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/89ab7024f78a4b02a0195ab3657e23b4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/89ab7024f78a4b02a0195ab3657e23b4 2024-11-13T18:26:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:26:08,123 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/C of ecb8a98d59975d334af6c23fb4444cd4 into 89ab7024f78a4b02a0195ab3657e23b4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:08,130 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:08,131 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/C, priority=13, startTime=1731522367840; duration=0sec 2024-11-13T18:26:08,131 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:08,132 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:C 2024-11-13T18:26:08,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522428292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:08,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522428309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:08,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522428418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:08,439 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/3feaab01e8954e31912d0c8034b7addf 2024-11-13T18:26:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:26:08,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/0ba5405df3964673936071e9024ada88 is 50, key is test_row_0/B:col10/1731522367326/Put/seqid=0 2024-11-13T18:26:08,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522428451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:08,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742352_1528 (size=12301) 2024-11-13T18:26:08,510 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/0ba5405df3964673936071e9024ada88 2024-11-13T18:26:08,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/556e728a03af4ca2a52b670581563939 is 50, key is test_row_0/C:col10/1731522367326/Put/seqid=0 2024-11-13T18:26:08,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742353_1529 (size=12301) 2024-11-13T18:26:08,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522428640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:08,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522428679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:08,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39384 deadline: 1731522428719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,728 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_1' in ecb8a98d59975d334af6c23fb4444cd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:08,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39340 deadline: 1731522428730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,755 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in ecb8a98d59975d334af6c23fb4444cd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39350 deadline: 1731522428730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,756 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in ecb8a98d59975d334af6c23fb4444cd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:08,869 INFO [Thread-2019 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2019. 2024-11-13T18:26:08,869 DEBUG [Thread-2019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:08,869 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:08,869 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:08,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:08,875 INFO [Thread-2021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2021. 2024-11-13T18:26:08,875 DEBUG [Thread-2021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:08,875 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:08,875 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:08,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:08,889 INFO [Thread-2013 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2013. 2024-11-13T18:26:08,889 DEBUG [Thread-2013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:08,889 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:08,889 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:08,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:08,891 INFO [Thread-2015 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2015. 2024-11-13T18:26:08,892 DEBUG [Thread-2015 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:08,892 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:08,892 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:08,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:08,894 INFO [Thread-2017 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2017. 2024-11-13T18:26:08,894 DEBUG [Thread-2017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:08,894 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:08,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:08,895 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:26:08,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39368 deadline: 1731522428955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:08,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39378 deadline: 1731522428997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., hostname=62ae331128c9,39615,1731522225114, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:26:09,011 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/556e728a03af4ca2a52b670581563939 2024-11-13T18:26:09,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/3feaab01e8954e31912d0c8034b7addf as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/3feaab01e8954e31912d0c8034b7addf 2024-11-13T18:26:09,041 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/3feaab01e8954e31912d0c8034b7addf, entries=150, sequenceid=394, filesize=12.0 K 2024-11-13T18:26:09,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/0ba5405df3964673936071e9024ada88 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0ba5405df3964673936071e9024ada88 2024-11-13T18:26:09,046 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0ba5405df3964673936071e9024ada88, entries=150, sequenceid=394, filesize=12.0 K 2024-11-13T18:26:09,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/556e728a03af4ca2a52b670581563939 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/556e728a03af4ca2a52b670581563939 2024-11-13T18:26:09,055 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/556e728a03af4ca2a52b670581563939, entries=150, sequenceid=394, filesize=12.0 K 2024-11-13T18:26:09,057 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ecb8a98d59975d334af6c23fb4444cd4 in 1098ms, sequenceid=394, compaction requested=false 2024-11-13T18:26:09,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:09,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:09,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-13T18:26:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-13T18:26:09,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-13T18:26:09,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2610 sec 2024-11-13T18:26:09,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.2830 sec 2024-11-13T18:26:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:09,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522368254, max timestamp=1731522369467Num uniques 859; 2024-11-13T18:26:09,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:26:09,475 INFO [Thread-2006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2006. 2024-11-13T18:26:09,475 DEBUG [Thread-2006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:09,475 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:09,475 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:09,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:09,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522368254, max timestamp=1731522369467Num uniques 859; 2024-11-13T18:26:09,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:26:09,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=57.35 KB heapSize=162.34 KB 2024-11-13T18:26:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:26:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:26:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:26:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:26:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:26:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:26:09,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/95757d962cc64617969c8d975f851eae is 50, key is test_row_0/A:col10/1731522368254/Put/seqid=0 2024-11-13T18:26:09,515 INFO [Thread-2004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2004. 2024-11-13T18:26:09,515 DEBUG [Thread-2004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:09,515 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:09,516 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:09,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:09,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742354_1530 (size=12301) 2024-11-13T18:26:09,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/95757d962cc64617969c8d975f851eae 2024-11-13T18:26:09,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/f13ce6d3776b47d6bb4eada9c3686628 is 50, key is test_row_0/B:col10/1731522368254/Put/seqid=0 2024-11-13T18:26:09,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742355_1531 (size=12301) 2024-11-13T18:26:09,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/f13ce6d3776b47d6bb4eada9c3686628 2024-11-13T18:26:09,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/fe1a8cca21014ab985543c2281cb42a5 is 50, key is test_row_0/C:col10/1731522368254/Put/seqid=0 2024-11-13T18:26:09,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742356_1532 (size=12301) 2024-11-13T18:26:09,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/fe1a8cca21014ab985543c2281cb42a5 2024-11-13T18:26:09,699 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:26:09,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/95757d962cc64617969c8d975f851eae as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/95757d962cc64617969c8d975f851eae 2024-11-13T18:26:09,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/95757d962cc64617969c8d975f851eae, entries=150, sequenceid=418, filesize=12.0 K 2024-11-13T18:26:09,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/f13ce6d3776b47d6bb4eada9c3686628 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f13ce6d3776b47d6bb4eada9c3686628 2024-11-13T18:26:09,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f13ce6d3776b47d6bb4eada9c3686628, entries=150, sequenceid=418, filesize=12.0 K 2024-11-13T18:26:09,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/fe1a8cca21014ab985543c2281cb42a5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/fe1a8cca21014ab985543c2281cb42a5 2024-11-13T18:26:09,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/fe1a8cca21014ab985543c2281cb42a5, entries=150, sequenceid=418, filesize=12.0 K 2024-11-13T18:26:09,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~57.35 KB/58730, heapSize ~162.30 KB/166192, currentSize=6.71 KB/6870 for ecb8a98d59975d334af6c23fb4444cd4 in 270ms, sequenceid=418, compaction requested=true 2024-11-13T18:26:09,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:09,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:09,747 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:09,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:09,747 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:09,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:09,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:09,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ecb8a98d59975d334af6c23fb4444cd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:09,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:09,748 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:09,748 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:09,748 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/B is initiating minor compaction (all files) 2024-11-13T18:26:09,748 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/A is initiating minor compaction (all files) 2024-11-13T18:26:09,748 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/B in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:09,748 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/A in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:09,748 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/da51c119185e43d4be72d2040fb9e9e5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/3feaab01e8954e31912d0c8034b7addf, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/95757d962cc64617969c8d975f851eae] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=36.8 K 2024-11-13T18:26:09,748 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/e11b0f00bc8941c983163a11a2034657, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0ba5405df3964673936071e9024ada88, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f13ce6d3776b47d6bb4eada9c3686628] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=36.8 K 2024-11-13T18:26:09,749 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e11b0f00bc8941c983163a11a2034657, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731522366466 2024-11-13T18:26:09,749 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting da51c119185e43d4be72d2040fb9e9e5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731522366466 2024-11-13T18:26:09,750 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ba5405df3964673936071e9024ada88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731522367308 2024-11-13T18:26:09,750 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3feaab01e8954e31912d0c8034b7addf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731522367308 2024-11-13T18:26:09,750 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting f13ce6d3776b47d6bb4eada9c3686628, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731522368254 2024-11-13T18:26:09,750 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95757d962cc64617969c8d975f851eae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731522368254 2024-11-13T18:26:09,777 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#A#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:09,777 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/573cb94e249e49e49f691208932d5116 is 50, key is test_row_0/A:col10/1731522368254/Put/seqid=0 2024-11-13T18:26:09,816 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#B#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:09,817 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/cc684d82135449ad9219f286672d3269 is 50, key is test_row_0/B:col10/1731522368254/Put/seqid=0 2024-11-13T18:26:09,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742357_1533 (size=13187) 2024-11-13T18:26:09,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742358_1534 (size=13187) 2024-11-13T18:26:09,910 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/573cb94e249e49e49f691208932d5116 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/573cb94e249e49e49f691208932d5116 2024-11-13T18:26:09,945 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/A of ecb8a98d59975d334af6c23fb4444cd4 into 573cb94e249e49e49f691208932d5116(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:09,945 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:09,945 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/A, priority=13, startTime=1731522369747; duration=0sec 2024-11-13T18:26:09,945 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:09,945 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:A 2024-11-13T18:26:09,945 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:09,947 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:09,947 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): ecb8a98d59975d334af6c23fb4444cd4/C is initiating minor compaction (all files) 2024-11-13T18:26:09,947 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ecb8a98d59975d334af6c23fb4444cd4/C in TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:09,948 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/89ab7024f78a4b02a0195ab3657e23b4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/556e728a03af4ca2a52b670581563939, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/fe1a8cca21014ab985543c2281cb42a5] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp, totalSize=36.8 K 2024-11-13T18:26:09,948 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89ab7024f78a4b02a0195ab3657e23b4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731522366466 2024-11-13T18:26:09,949 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 556e728a03af4ca2a52b670581563939, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731522367308 2024-11-13T18:26:09,949 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1a8cca21014ab985543c2281cb42a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731522368254 2024-11-13T18:26:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:26:09,960 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:09,980 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ecb8a98d59975d334af6c23fb4444cd4#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:09,981 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/3d36e68f0a6449f6ad91b839f44fd8c8 is 50, key is test_row_0/C:col10/1731522368254/Put/seqid=0 2024-11-13T18:26:10,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742359_1535 (size=13187) 2024-11-13T18:26:10,034 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/3d36e68f0a6449f6ad91b839f44fd8c8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/3d36e68f0a6449f6ad91b839f44fd8c8 2024-11-13T18:26:10,040 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/C of ecb8a98d59975d334af6c23fb4444cd4 into 3d36e68f0a6449f6ad91b839f44fd8c8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:10,040 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:10,040 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/C, priority=13, startTime=1731522369747; duration=0sec 2024-11-13T18:26:10,041 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:10,041 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:C 2024-11-13T18:26:10,299 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/cc684d82135449ad9219f286672d3269 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/cc684d82135449ad9219f286672d3269 2024-11-13T18:26:10,308 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ecb8a98d59975d334af6c23fb4444cd4/B of ecb8a98d59975d334af6c23fb4444cd4 into cc684d82135449ad9219f286672d3269(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:10,308 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ecb8a98d59975d334af6c23fb4444cd4: 2024-11-13T18:26:10,308 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., storeName=ecb8a98d59975d334af6c23fb4444cd4/B, priority=13, startTime=1731522369747; duration=0sec 2024-11-13T18:26:10,308 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:10,308 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ecb8a98d59975d334af6c23fb4444cd4:B 2024-11-13T18:26:10,763 INFO [Thread-2010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2010. 2024-11-13T18:26:10,763 DEBUG [Thread-2010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:10,763 DEBUG [Thread-2010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:10,764 DEBUG [Thread-2010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:10,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:10,780 INFO [Thread-2002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2002. 2024-11-13T18:26:10,780 DEBUG [Thread-2002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:10,780 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:10,780 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:10,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:10,788 INFO [Thread-2008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2008. 2024-11-13T18:26:10,788 DEBUG [Thread-2008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:10,788 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:10,788 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:10,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:10,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 579 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1737 rows 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 582 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1746 rows 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 581 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1743 rows 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 596 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1746 rows 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 583 2024-11-13T18:26:10,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1749 rows 2024-11-13T18:26:10,791 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:26:10,791 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:26:10,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:10,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:10,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:10,793 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:26:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:26:10,797 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522370797"}]},"ts":"1731522370797"} 2024-11-13T18:26:10,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:26:10,799 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:26:10,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:26:10,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ecb8a98d59975d334af6c23fb4444cd4, UNASSIGN}] 2024-11-13T18:26:10,803 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ecb8a98d59975d334af6c23fb4444cd4, UNASSIGN 2024-11-13T18:26:10,804 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=ecb8a98d59975d334af6c23fb4444cd4, regionState=CLOSING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:26:10,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ecb8a98d59975d334af6c23fb4444cd4, UNASSIGN because future has completed 2024-11-13T18:26:10,807 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:26:10,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:26:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:26:10,967 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(122): Close ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:10,967 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:26:10,968 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1722): Closing ecb8a98d59975d334af6c23fb4444cd4, disabling compactions & flushes 2024-11-13T18:26:10,968 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:10,968 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:10,968 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. after waiting 0 ms 2024-11-13T18:26:10,968 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:10,968 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(2902): Flushing ecb8a98d59975d334af6c23fb4444cd4 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T18:26:10,968 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=A 2024-11-13T18:26:10,968 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:10,968 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=B 2024-11-13T18:26:10,968 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:10,968 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ecb8a98d59975d334af6c23fb4444cd4, store=C 2024-11-13T18:26:10,968 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:10,981 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/6343ce16fd3d47c683df71c9452b3147 is 50, key is test_row_0/A:col10/1731522370786/Put/seqid=0 2024-11-13T18:26:11,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742360_1536 (size=12301) 2024-11-13T18:26:11,039 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/6343ce16fd3d47c683df71c9452b3147 2024-11-13T18:26:11,059 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/221e592a1ebf4c1fb04682603f99eb2c is 50, key is test_row_0/B:col10/1731522370786/Put/seqid=0 2024-11-13T18:26:11,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742361_1537 (size=12301) 2024-11-13T18:26:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:26:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:26:11,507 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/221e592a1ebf4c1fb04682603f99eb2c 2024-11-13T18:26:11,573 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/a4b351ce205a477580e44e82ef420c6a is 50, key is test_row_0/C:col10/1731522370786/Put/seqid=0 2024-11-13T18:26:11,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742362_1538 (size=12301) 2024-11-13T18:26:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:26:12,011 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/a4b351ce205a477580e44e82ef420c6a 2024-11-13T18:26:12,034 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/A/6343ce16fd3d47c683df71c9452b3147 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/6343ce16fd3d47c683df71c9452b3147 2024-11-13T18:26:12,045 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/6343ce16fd3d47c683df71c9452b3147, entries=150, sequenceid=428, filesize=12.0 K 2024-11-13T18:26:12,055 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/B/221e592a1ebf4c1fb04682603f99eb2c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/221e592a1ebf4c1fb04682603f99eb2c 2024-11-13T18:26:12,085 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/221e592a1ebf4c1fb04682603f99eb2c, entries=150, sequenceid=428, filesize=12.0 K 2024-11-13T18:26:12,085 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/.tmp/C/a4b351ce205a477580e44e82ef420c6a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/a4b351ce205a477580e44e82ef420c6a 2024-11-13T18:26:12,100 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/a4b351ce205a477580e44e82ef420c6a, entries=150, sequenceid=428, filesize=12.0 K 2024-11-13T18:26:12,102 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ecb8a98d59975d334af6c23fb4444cd4 in 1133ms, sequenceid=428, compaction requested=false 2024-11-13T18:26:12,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/d9a2cd9aa0824e85afebfa410754255f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/0acd406385cd42eda83d05e65e4fbee3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/99e98b792fc64775a6f5a2146298654a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b089e14d7cd5494fa07b11f82c6a04c0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/ba8b8eceab144060a4e4becfa84edc35, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/4d34beb8d3fd459e8b5f18af156d7f43, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/5bf6c684f919418f8bd2fc62527a6468, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/7209d9393c674dce9e2143d71b065f88, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/8b541161fe7b4297b247890ac31fe212, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b063d0f3fe964d3db9fb0ccb2d320c52, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/6ed7e8bdc3534656930dca7cfc1f78f6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/29b7ff6c6b524560941ed97ef1ec32cc, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/278ff840142941f9b462a7059c6996a3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2c27cc8e6e01473ea0b5527adc0315c3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b8cab50f3bae42db97fe7199b8fbe2a2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/caca4492c76c4898b8abeb9b62f32ec9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/68e36445aa3e41c5807e92f57605972b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/532484058dc2449884c72cd1f54fef76, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/17f1d8c9781b4a2d9fbb380cb89a2f48, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/51763102abe74d2685b98145fe8693fb, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/9fc038f108a04df2b7feda358124468f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/833ff12cdd0e4003971b38af7040bab1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/e02c465a98894296a3a0e79500008ae5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/da51c119185e43d4be72d2040fb9e9e5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/3feaab01e8954e31912d0c8034b7addf, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/95757d962cc64617969c8d975f851eae] to archive 2024-11-13T18:26:12,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:26:12,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/d9a2cd9aa0824e85afebfa410754255f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/d9a2cd9aa0824e85afebfa410754255f 2024-11-13T18:26:12,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/0acd406385cd42eda83d05e65e4fbee3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/0acd406385cd42eda83d05e65e4fbee3 2024-11-13T18:26:12,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/99e98b792fc64775a6f5a2146298654a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/99e98b792fc64775a6f5a2146298654a 2024-11-13T18:26:12,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b089e14d7cd5494fa07b11f82c6a04c0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b089e14d7cd5494fa07b11f82c6a04c0 2024-11-13T18:26:12,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/ba8b8eceab144060a4e4becfa84edc35 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/ba8b8eceab144060a4e4becfa84edc35 2024-11-13T18:26:12,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/4d34beb8d3fd459e8b5f18af156d7f43 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/4d34beb8d3fd459e8b5f18af156d7f43 2024-11-13T18:26:12,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/5bf6c684f919418f8bd2fc62527a6468 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/5bf6c684f919418f8bd2fc62527a6468 2024-11-13T18:26:12,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/7209d9393c674dce9e2143d71b065f88 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/7209d9393c674dce9e2143d71b065f88 2024-11-13T18:26:12,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/8b541161fe7b4297b247890ac31fe212 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/8b541161fe7b4297b247890ac31fe212 2024-11-13T18:26:12,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b063d0f3fe964d3db9fb0ccb2d320c52 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b063d0f3fe964d3db9fb0ccb2d320c52 2024-11-13T18:26:12,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/6ed7e8bdc3534656930dca7cfc1f78f6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/6ed7e8bdc3534656930dca7cfc1f78f6 2024-11-13T18:26:12,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/29b7ff6c6b524560941ed97ef1ec32cc to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/29b7ff6c6b524560941ed97ef1ec32cc 2024-11-13T18:26:12,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/278ff840142941f9b462a7059c6996a3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/278ff840142941f9b462a7059c6996a3 2024-11-13T18:26:12,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2c27cc8e6e01473ea0b5527adc0315c3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/2c27cc8e6e01473ea0b5527adc0315c3 2024-11-13T18:26:12,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b8cab50f3bae42db97fe7199b8fbe2a2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/b8cab50f3bae42db97fe7199b8fbe2a2 2024-11-13T18:26:12,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/caca4492c76c4898b8abeb9b62f32ec9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/caca4492c76c4898b8abeb9b62f32ec9 2024-11-13T18:26:12,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/68e36445aa3e41c5807e92f57605972b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/68e36445aa3e41c5807e92f57605972b 2024-11-13T18:26:12,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/532484058dc2449884c72cd1f54fef76 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/532484058dc2449884c72cd1f54fef76 2024-11-13T18:26:12,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/17f1d8c9781b4a2d9fbb380cb89a2f48 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/17f1d8c9781b4a2d9fbb380cb89a2f48 2024-11-13T18:26:12,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/51763102abe74d2685b98145fe8693fb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/51763102abe74d2685b98145fe8693fb 2024-11-13T18:26:12,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/9fc038f108a04df2b7feda358124468f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/9fc038f108a04df2b7feda358124468f 2024-11-13T18:26:12,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/833ff12cdd0e4003971b38af7040bab1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/833ff12cdd0e4003971b38af7040bab1 2024-11-13T18:26:12,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/e02c465a98894296a3a0e79500008ae5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/e02c465a98894296a3a0e79500008ae5 2024-11-13T18:26:12,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/da51c119185e43d4be72d2040fb9e9e5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/da51c119185e43d4be72d2040fb9e9e5 2024-11-13T18:26:12,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/3feaab01e8954e31912d0c8034b7addf to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/3feaab01e8954e31912d0c8034b7addf 2024-11-13T18:26:12,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/95757d962cc64617969c8d975f851eae to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/95757d962cc64617969c8d975f851eae 2024-11-13T18:26:12,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/9f02b34001df4c8d83a5071b0295731c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/948e952df50f464f932ff88e6fa264a0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5d526a2679f04c06a0a533a5569f4357, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/69b074555d5e4514861382bf9a6890d1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d8cf2dd856504bf4b3154c4dc6d39e07, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/644f6bdc8aef4e41b0e21fd0ca4fa437, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/74443ba015dd404485a3bc12f0b842db, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/39170ff8ca514ba490ec1ca8480c2452, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d3d759b574284391adbbeaa4f9fd87da, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3b249cf47bf4f4d8dc736cd77dbefac, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f8db36d6793a4ceb85d3e53179b1a9b3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3d9a0b69fac4b89911828814e67598c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/bac7027d9f2a48e183b67a86323dcb88, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5befdb024c134e248580805bafa2be69, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0349d23281ee4b3297c22f91db2b95ad, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0dc95a534a454f48ad0beff6679cf6ed, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b4c7b72a7213430381675a500f7e7018, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8b99627fc3fb4c7d8951a5a3d85ec7f9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d2256f7bda8e44ae893a754e49178a32, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5743710ade874b95af0783e9b750fa28, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1b0e721896f742d297233ea9f124a9a9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c04ca5067e17488b80acb3a9f5b01b84, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/e11b0f00bc8941c983163a11a2034657, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1a4a7d06138747a9a68060c5295396e5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0ba5405df3964673936071e9024ada88, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f13ce6d3776b47d6bb4eada9c3686628] to archive 2024-11-13T18:26:12,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:26:12,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/9f02b34001df4c8d83a5071b0295731c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/9f02b34001df4c8d83a5071b0295731c 2024-11-13T18:26:12,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/948e952df50f464f932ff88e6fa264a0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/948e952df50f464f932ff88e6fa264a0 2024-11-13T18:26:12,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5d526a2679f04c06a0a533a5569f4357 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5d526a2679f04c06a0a533a5569f4357 2024-11-13T18:26:12,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/69b074555d5e4514861382bf9a6890d1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/69b074555d5e4514861382bf9a6890d1 2024-11-13T18:26:12,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d8cf2dd856504bf4b3154c4dc6d39e07 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d8cf2dd856504bf4b3154c4dc6d39e07 2024-11-13T18:26:12,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/644f6bdc8aef4e41b0e21fd0ca4fa437 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/644f6bdc8aef4e41b0e21fd0ca4fa437 2024-11-13T18:26:12,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/74443ba015dd404485a3bc12f0b842db to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/74443ba015dd404485a3bc12f0b842db 2024-11-13T18:26:12,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/39170ff8ca514ba490ec1ca8480c2452 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/39170ff8ca514ba490ec1ca8480c2452 2024-11-13T18:26:12,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d3d759b574284391adbbeaa4f9fd87da to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d3d759b574284391adbbeaa4f9fd87da 2024-11-13T18:26:12,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3b249cf47bf4f4d8dc736cd77dbefac to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3b249cf47bf4f4d8dc736cd77dbefac 2024-11-13T18:26:12,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f8db36d6793a4ceb85d3e53179b1a9b3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f8db36d6793a4ceb85d3e53179b1a9b3 2024-11-13T18:26:12,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3d9a0b69fac4b89911828814e67598c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c3d9a0b69fac4b89911828814e67598c 2024-11-13T18:26:12,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/bac7027d9f2a48e183b67a86323dcb88 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/bac7027d9f2a48e183b67a86323dcb88 2024-11-13T18:26:12,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5befdb024c134e248580805bafa2be69 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5befdb024c134e248580805bafa2be69 2024-11-13T18:26:12,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0349d23281ee4b3297c22f91db2b95ad to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0349d23281ee4b3297c22f91db2b95ad 2024-11-13T18:26:12,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0dc95a534a454f48ad0beff6679cf6ed to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0dc95a534a454f48ad0beff6679cf6ed 2024-11-13T18:26:12,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b4c7b72a7213430381675a500f7e7018 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/b4c7b72a7213430381675a500f7e7018 2024-11-13T18:26:12,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8b99627fc3fb4c7d8951a5a3d85ec7f9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/8b99627fc3fb4c7d8951a5a3d85ec7f9 2024-11-13T18:26:12,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d2256f7bda8e44ae893a754e49178a32 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/d2256f7bda8e44ae893a754e49178a32 2024-11-13T18:26:12,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5743710ade874b95af0783e9b750fa28 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/5743710ade874b95af0783e9b750fa28 2024-11-13T18:26:12,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1b0e721896f742d297233ea9f124a9a9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1b0e721896f742d297233ea9f124a9a9 2024-11-13T18:26:12,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c04ca5067e17488b80acb3a9f5b01b84 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/c04ca5067e17488b80acb3a9f5b01b84 2024-11-13T18:26:12,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/e11b0f00bc8941c983163a11a2034657 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/e11b0f00bc8941c983163a11a2034657 2024-11-13T18:26:12,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1a4a7d06138747a9a68060c5295396e5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/1a4a7d06138747a9a68060c5295396e5 2024-11-13T18:26:12,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0ba5405df3964673936071e9024ada88 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/0ba5405df3964673936071e9024ada88 2024-11-13T18:26:12,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f13ce6d3776b47d6bb4eada9c3686628 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/f13ce6d3776b47d6bb4eada9c3686628 2024-11-13T18:26:12,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/ed64d6261ef24b1780b68619cd628685, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7939285d0032422abf0a5ba0fc8b2bee, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/3999ca816dc44200a2f06f72b85dbc12, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/402947b313184853b682a2cd2552ed37, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2d6e85a1fdf94e41a86df5bb3b7c7739, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/0031cc22128d4d6bb25de2f5acc085c4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e63a75c39be4423a8f05539f1b248b88, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/bf9470bd1da24010906341e57f54201b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7540bd9559434dbea5b1b8495e6dbe5b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/c3a65affe4874016a0d43008f4ea8d17, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/b28dfc2530ff4d14a799bd1d12c3f80a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e30fb7beef5a43a896a6d22fd16fcc8e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7d25205631d141cea564705776b49e34, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/777edf2c361a4566bbce8f640ad5d1d8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/9d2c5a3d9ea2438aa8ab101051b94746, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e688e484c343475884e88dab8af5d290, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abeb50403644555b172eef7b19c68aa, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2e9c313bd9e54dea8ff31507487ae229, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abac2b288e6440995bace61f4a151ee, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5ab01c3aaace49578f2d1846864e338a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/8f5567b297294d669fa05ef9838a71f1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/238b5ebd71574b0d9254e3085ed05499, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/89ab7024f78a4b02a0195ab3657e23b4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/98dc195a24a1490daf5fc01dddf693f8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/556e728a03af4ca2a52b670581563939, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/fe1a8cca21014ab985543c2281cb42a5] to archive 2024-11-13T18:26:12,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:26:12,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/ed64d6261ef24b1780b68619cd628685 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/ed64d6261ef24b1780b68619cd628685 2024-11-13T18:26:12,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7939285d0032422abf0a5ba0fc8b2bee to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7939285d0032422abf0a5ba0fc8b2bee 2024-11-13T18:26:12,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/3999ca816dc44200a2f06f72b85dbc12 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/3999ca816dc44200a2f06f72b85dbc12 2024-11-13T18:26:12,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/402947b313184853b682a2cd2552ed37 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/402947b313184853b682a2cd2552ed37 2024-11-13T18:26:12,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2d6e85a1fdf94e41a86df5bb3b7c7739 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2d6e85a1fdf94e41a86df5bb3b7c7739 2024-11-13T18:26:12,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/0031cc22128d4d6bb25de2f5acc085c4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/0031cc22128d4d6bb25de2f5acc085c4 2024-11-13T18:26:12,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e63a75c39be4423a8f05539f1b248b88 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e63a75c39be4423a8f05539f1b248b88 2024-11-13T18:26:12,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/bf9470bd1da24010906341e57f54201b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/bf9470bd1da24010906341e57f54201b 2024-11-13T18:26:12,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7540bd9559434dbea5b1b8495e6dbe5b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7540bd9559434dbea5b1b8495e6dbe5b 2024-11-13T18:26:12,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/c3a65affe4874016a0d43008f4ea8d17 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/c3a65affe4874016a0d43008f4ea8d17 2024-11-13T18:26:12,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/b28dfc2530ff4d14a799bd1d12c3f80a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/b28dfc2530ff4d14a799bd1d12c3f80a 2024-11-13T18:26:12,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e30fb7beef5a43a896a6d22fd16fcc8e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e30fb7beef5a43a896a6d22fd16fcc8e 2024-11-13T18:26:12,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7d25205631d141cea564705776b49e34 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/7d25205631d141cea564705776b49e34 2024-11-13T18:26:12,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/777edf2c361a4566bbce8f640ad5d1d8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/777edf2c361a4566bbce8f640ad5d1d8 2024-11-13T18:26:12,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/9d2c5a3d9ea2438aa8ab101051b94746 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/9d2c5a3d9ea2438aa8ab101051b94746 2024-11-13T18:26:12,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e688e484c343475884e88dab8af5d290 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/e688e484c343475884e88dab8af5d290 2024-11-13T18:26:12,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abeb50403644555b172eef7b19c68aa to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abeb50403644555b172eef7b19c68aa 2024-11-13T18:26:12,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2e9c313bd9e54dea8ff31507487ae229 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/2e9c313bd9e54dea8ff31507487ae229 2024-11-13T18:26:12,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abac2b288e6440995bace61f4a151ee to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5abac2b288e6440995bace61f4a151ee 2024-11-13T18:26:12,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5ab01c3aaace49578f2d1846864e338a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/5ab01c3aaace49578f2d1846864e338a 2024-11-13T18:26:12,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/8f5567b297294d669fa05ef9838a71f1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/8f5567b297294d669fa05ef9838a71f1 2024-11-13T18:26:12,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/238b5ebd71574b0d9254e3085ed05499 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/238b5ebd71574b0d9254e3085ed05499 2024-11-13T18:26:12,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/89ab7024f78a4b02a0195ab3657e23b4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/89ab7024f78a4b02a0195ab3657e23b4 2024-11-13T18:26:12,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/98dc195a24a1490daf5fc01dddf693f8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/98dc195a24a1490daf5fc01dddf693f8 2024-11-13T18:26:12,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/556e728a03af4ca2a52b670581563939 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/556e728a03af4ca2a52b670581563939 2024-11-13T18:26:12,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/fe1a8cca21014ab985543c2281cb42a5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/fe1a8cca21014ab985543c2281cb42a5 2024-11-13T18:26:12,272 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/recovered.edits/431.seqid, newMaxSeqId=431, maxSeqId=1 2024-11-13T18:26:12,273 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. 2024-11-13T18:26:12,273 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1676): Region close journal for ecb8a98d59975d334af6c23fb4444cd4: Waiting for close lock at 1731522370967Running coprocessor pre-close hooks at 1731522370967Disabling compacts and flushes for region at 1731522370967Disabling writes for close at 1731522370968 (+1 ms)Obtaining lock to block concurrent updates at 1731522370968Preparing flush snapshotting stores in ecb8a98d59975d334af6c23fb4444cd4 at 1731522370968Finished memstore snapshotting TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731522370969 (+1 ms)Flushing stores of TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4. at 1731522370969Flushing ecb8a98d59975d334af6c23fb4444cd4/A: creating writer at 1731522370969Flushing ecb8a98d59975d334af6c23fb4444cd4/A: appending metadata at 1731522370980 (+11 ms)Flushing ecb8a98d59975d334af6c23fb4444cd4/A: closing flushed file at 1731522370980Flushing ecb8a98d59975d334af6c23fb4444cd4/B: creating writer at 1731522371049 (+69 ms)Flushing ecb8a98d59975d334af6c23fb4444cd4/B: appending metadata at 1731522371059 (+10 ms)Flushing ecb8a98d59975d334af6c23fb4444cd4/B: closing flushed file at 1731522371059Flushing ecb8a98d59975d334af6c23fb4444cd4/C: creating writer at 1731522371549 (+490 ms)Flushing ecb8a98d59975d334af6c23fb4444cd4/C: appending metadata at 1731522371572 (+23 ms)Flushing ecb8a98d59975d334af6c23fb4444cd4/C: closing flushed file at 1731522371572Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51066f93: reopening flushed file at 1731522372033 (+461 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32744c06: reopening flushed file at 1731522372045 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ac1c595: reopening flushed file at 1731522372085 (+40 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ecb8a98d59975d334af6c23fb4444cd4 in 1133ms, sequenceid=428, compaction requested=false at 1731522372102 (+17 ms)Writing region close event to WAL at 1731522372263 (+161 ms)Running coprocessor post-close hooks at 1731522372273 (+10 ms)Closed at 1731522372273 2024-11-13T18:26:12,287 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(157): Closed ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:12,297 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=ecb8a98d59975d334af6c23fb4444cd4, regionState=CLOSED 2024-11-13T18:26:12,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:26:12,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-13T18:26:12,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseRegionProcedure ecb8a98d59975d334af6c23fb4444cd4, server=62ae331128c9,39615,1731522225114 in 1.5020 sec 2024-11-13T18:26:12,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-13T18:26:12,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ecb8a98d59975d334af6c23fb4444cd4, UNASSIGN in 1.5100 sec 2024-11-13T18:26:12,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-13T18:26:12,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5160 sec 2024-11-13T18:26:12,322 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522372322"}]},"ts":"1731522372322"} 2024-11-13T18:26:12,324 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:26:12,324 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:26:12,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-13T18:26:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:26:12,947 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:12,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:26:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:12,959 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:26:12,961 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=125, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:12,971 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:13,012 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/recovered.edits] 2024-11-13T18:26:13,027 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/573cb94e249e49e49f691208932d5116 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/573cb94e249e49e49f691208932d5116 2024-11-13T18:26:13,039 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/6343ce16fd3d47c683df71c9452b3147 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/A/6343ce16fd3d47c683df71c9452b3147 2024-11-13T18:26:13,055 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/221e592a1ebf4c1fb04682603f99eb2c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/221e592a1ebf4c1fb04682603f99eb2c 2024-11-13T18:26:13,063 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/cc684d82135449ad9219f286672d3269 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/B/cc684d82135449ad9219f286672d3269 2024-11-13T18:26:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:26:13,075 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/3d36e68f0a6449f6ad91b839f44fd8c8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/3d36e68f0a6449f6ad91b839f44fd8c8 2024-11-13T18:26:13,092 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/a4b351ce205a477580e44e82ef420c6a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/C/a4b351ce205a477580e44e82ef420c6a 2024-11-13T18:26:13,105 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/recovered.edits/431.seqid to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4/recovered.edits/431.seqid 2024-11-13T18:26:13,107 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/ecb8a98d59975d334af6c23fb4444cd4 2024-11-13T18:26:13,110 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:26:13,127 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=125, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:13,130 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:26:13,132 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:26:13,136 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=125, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:13,137 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:26:13,140 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731522373137"}]},"ts":"9223372036854775807"} 2024-11-13T18:26:13,143 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:26:13,144 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ecb8a98d59975d334af6c23fb4444cd4, NAME => 'TestAcidGuarantees,,1731522347495.ecb8a98d59975d334af6c23fb4444cd4.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:26:13,144 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:26:13,144 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731522373144"}]},"ts":"9223372036854775807"} 2024-11-13T18:26:13,148 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:26:13,149 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=125, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:13,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 201 msec 2024-11-13T18:26:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:26:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:26:13,277 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:13,295 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=455 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1913 (was 1955), ProcessCount=11 (was 11), AvailableMemoryMB=1778 (was 1780) 2024-11-13T18:26:13,308 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=1913, ProcessCount=11, AvailableMemoryMB=1777 2024-11-13T18:26:13,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:26:13,309 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:26:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:13,312 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:26:13,312 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:13,313 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:26:13,314 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 126 2024-11-13T18:26:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-13T18:26:13,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742363_1539 (size=1061) 2024-11-13T18:26:13,374 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b 2024-11-13T18:26:13,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742364_1540 (size=53) 2024-11-13T18:26:13,405 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:13,405 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2bf982671ac78b0bc57c6ef3a6b571eb, disabling compactions & flushes 2024-11-13T18:26:13,405 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:13,405 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:13,406 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. after waiting 0 ms 2024-11-13T18:26:13,406 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:13,406 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:13,406 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2bf982671ac78b0bc57c6ef3a6b571eb: Waiting for close lock at 1731522373405Disabling compacts and flushes for region at 1731522373405Disabling writes for close at 1731522373406 (+1 ms)Writing region close event to WAL at 1731522373406Closed at 1731522373406 2024-11-13T18:26:13,407 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:26:13,408 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731522373407"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731522373407"}]},"ts":"1731522373407"} 2024-11-13T18:26:13,412 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:26:13,413 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:26:13,413 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522373413"}]},"ts":"1731522373413"} 2024-11-13T18:26:13,416 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:26:13,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, ASSIGN}] 2024-11-13T18:26:13,419 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, ASSIGN 2024-11-13T18:26:13,423 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, ASSIGN; state=OFFLINE, location=62ae331128c9,39615,1731522225114; forceNewPlan=false, retain=false 2024-11-13T18:26:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-13T18:26:13,478 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:26:13,574 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=2bf982671ac78b0bc57c6ef3a6b571eb, regionState=OPENING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:26:13,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, ASSIGN because future has completed 2024-11-13T18:26:13,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:26:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-13T18:26:13,743 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:13,743 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7752): Opening region: {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:26:13,744 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,744 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:13,744 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7794): checking encryption for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,744 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7797): checking classloading for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,763 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,779 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:26:13,779 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bf982671ac78b0bc57c6ef3a6b571eb columnFamilyName A 2024-11-13T18:26:13,779 DEBUG [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:13,781 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(327): Store=2bf982671ac78b0bc57c6ef3a6b571eb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:13,781 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,795 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:26:13,795 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bf982671ac78b0bc57c6ef3a6b571eb columnFamilyName B 2024-11-13T18:26:13,795 DEBUG [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:13,799 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(327): Store=2bf982671ac78b0bc57c6ef3a6b571eb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:13,800 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,811 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:26:13,811 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bf982671ac78b0bc57c6ef3a6b571eb columnFamilyName C 2024-11-13T18:26:13,811 DEBUG [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:13,819 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(327): Store=2bf982671ac78b0bc57c6ef3a6b571eb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:13,823 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:13,823 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1038): replaying wal for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,824 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,830 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,837 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1048): stopping wal replay for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,837 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1060): Cleaning up temporary data for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,839 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:26:13,840 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1093): writing seq id for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,854 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:26:13,863 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1114): Opened 2bf982671ac78b0bc57c6ef3a6b571eb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68159565, jitterRate=0.015656664967536926}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:26:13,863 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:13,866 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1006): Region open journal for 2bf982671ac78b0bc57c6ef3a6b571eb: Running coprocessor pre-open hook at 1731522373744Writing region info on filesystem at 1731522373744Initializing all the Stores at 1731522373755 (+11 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522373756 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522373758 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522373758Cleaning up temporary data from old regions at 1731522373837 (+79 ms)Running coprocessor post-open hooks at 1731522373863 (+26 ms)Region opened successfully at 1731522373864 (+1 ms) 2024-11-13T18:26:13,871 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., pid=128, masterSystemTime=1731522373729 2024-11-13T18:26:13,876 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=2bf982671ac78b0bc57c6ef3a6b571eb, regionState=OPEN, openSeqNum=2, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:26:13,878 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:13,880 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:13,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:26:13,881 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45483 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=62ae331128c9,39615,1731522225114, table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:26:13,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-13T18:26:13,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; OpenRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 in 305 msec 2024-11-13T18:26:13,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-13T18:26:13,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, ASSIGN in 469 msec 2024-11-13T18:26:13,891 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:26:13,891 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522373891"}]},"ts":"1731522373891"} 2024-11-13T18:26:13,893 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:26:13,894 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:26:13,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 585 msec 2024-11-13T18:26:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-13T18:26:13,949 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:13,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e177505, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:13,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:13,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:13,952 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67cf0150, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:13,955 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:13,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219b0d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:13,959 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:13,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:13,966 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:26:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 62ae331128c9,45483,1731522221514 2024-11-13T18:26:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@41e58aac 2024-11-13T18:26:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:26:13,969 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:26:13,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:26:13,972 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:26:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:14,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742365_1541 (size=1097) 2024-11-13T18:26:14,076 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T18:26:14,076 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T18:26:14,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:26:14,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, REOPEN/MOVE}] 2024-11-13T18:26:14,086 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, REOPEN/MOVE 2024-11-13T18:26:14,087 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=2bf982671ac78b0bc57c6ef3a6b571eb, regionState=CLOSING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:26:14,093 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45483 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=62ae331128c9,39615,1731522225114, table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:26:14,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, REOPEN/MOVE because future has completed 2024-11-13T18:26:14,095 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:26:14,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:26:14,259 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,260 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:26:14,260 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing 2bf982671ac78b0bc57c6ef3a6b571eb, disabling compactions & flushes 2024-11-13T18:26:14,260 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:14,260 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:14,260 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. after waiting 0 ms 2024-11-13T18:26:14,260 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:14,284 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:26:14,291 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:14,291 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for 2bf982671ac78b0bc57c6ef3a6b571eb: Waiting for close lock at 1731522374260Running coprocessor pre-close hooks at 1731522374260Disabling compacts and flushes for region at 1731522374260Disabling writes for close at 1731522374260Writing region close event to WAL at 1731522374261 (+1 ms)Running coprocessor post-close hooks at 1731522374291 (+30 ms)Closed at 1731522374291 2024-11-13T18:26:14,291 WARN [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionServer(3299): Not adding moved region record: 2bf982671ac78b0bc57c6ef3a6b571eb to self. 2024-11-13T18:26:14,294 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,294 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=2bf982671ac78b0bc57c6ef3a6b571eb, regionState=CLOSED 2024-11-13T18:26:14,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:26:14,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-13T18:26:14,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 in 204 msec 2024-11-13T18:26:14,306 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, REOPEN/MOVE; state=CLOSED, location=62ae331128c9,39615,1731522225114; forceNewPlan=false, retain=true 2024-11-13T18:26:14,457 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=2bf982671ac78b0bc57c6ef3a6b571eb, regionState=OPENING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:26:14,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, REOPEN/MOVE because future has completed 2024-11-13T18:26:14,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:26:14,645 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:14,645 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(7752): Opening region: {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:26:14,646 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,646 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:14,646 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(7794): checking encryption for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,646 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(7797): checking classloading for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,651 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,655 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:26:14,655 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bf982671ac78b0bc57c6ef3a6b571eb columnFamilyName A 2024-11-13T18:26:14,668 DEBUG [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:14,671 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(327): Store=2bf982671ac78b0bc57c6ef3a6b571eb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:14,672 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,681 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:26:14,681 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bf982671ac78b0bc57c6ef3a6b571eb columnFamilyName B 2024-11-13T18:26:14,681 DEBUG [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:14,687 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(327): Store=2bf982671ac78b0bc57c6ef3a6b571eb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:14,687 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,690 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T18:26:14,690 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bf982671ac78b0bc57c6ef3a6b571eb columnFamilyName C 2024-11-13T18:26:14,690 DEBUG [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:14,699 INFO [StoreOpener-2bf982671ac78b0bc57c6ef3a6b571eb-1 {}] regionserver.HStore(327): Store=2bf982671ac78b0bc57c6ef3a6b571eb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:14,699 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:14,699 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(1038): replaying wal for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,706 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,712 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,715 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(1048): stopping wal replay for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,715 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(1060): Cleaning up temporary data for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,716 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:26:14,717 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(1093): writing seq id for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,718 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(1114): Opened 2bf982671ac78b0bc57c6ef3a6b571eb; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66262832, jitterRate=-0.01260685920715332}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:26:14,718 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,719 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(1006): Region open journal for 2bf982671ac78b0bc57c6ef3a6b571eb: Running coprocessor pre-open hook at 1731522374646Writing region info on filesystem at 1731522374646Initializing all the Stores at 1731522374648 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522374648Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522374649 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522374649Cleaning up temporary data from old regions at 1731522374715 (+66 ms)Running coprocessor post-open hooks at 1731522374718 (+3 ms)Region opened successfully at 1731522374719 (+1 ms) 2024-11-13T18:26:14,725 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., pid=133, masterSystemTime=1731522374622 2024-11-13T18:26:14,735 DEBUG [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:14,736 INFO [RS_OPEN_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:14,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=2bf982671ac78b0bc57c6ef3a6b571eb, regionState=OPEN, openSeqNum=5, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:26:14,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:26:14,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=131 2024-11-13T18:26:14,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=131, state=SUCCESS, hasLock=false; OpenRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 in 305 msec 2024-11-13T18:26:14,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-13T18:26:14,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, REOPEN/MOVE in 686 msec 2024-11-13T18:26:14,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 815 msec 2024-11-13T18:26:14,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-13T18:26:14,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 693 msec 2024-11-13T18:26:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:26:14,791 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:14,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667ac2a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:14,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:14,796 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8dc7f4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:14,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:14,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:14,804 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e0681, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc3c2e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:14,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:14,813 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fad6196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:14,819 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dd5c324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325cce9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:14,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:14,840 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25a42bd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:14,842 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24940c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763f45be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:14,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:14,852 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f430d93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:14,855 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0d03ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0c0fb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:14,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:14,877 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47277841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:14,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:14,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528ca68b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@608c40ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:14,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:14,884 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:14,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:14,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:14,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e40888e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:14,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:14,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:14,887 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:14,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16929525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4010fbb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:14,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:14,892 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:14,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:14,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:14,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17dc65cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:14,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:14,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:14,896 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c10759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@334a63bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:14,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:14,900 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37019104, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:14,902 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d46a8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a39180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:14,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:14,922 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:14,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39794a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:14,924 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:14,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd41671, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1784f552, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,939 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 62ae331128c9,45483,-1 for getting cluster id 2024-11-13T18:26:14,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:26:14,942 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5912eaa4-9875-4022-99ef-f25a03d8a257' 2024-11-13T18:26:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:26:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5912eaa4-9875-4022-99ef-f25a03d8a257" 2024-11-13T18:26:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b04d3e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [62ae331128c9,45483,-1] 2024-11-13T18:26:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:26:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:14,952 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:26:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf340fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:26:14,954 DEBUG [Thread-2416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:14,955 DEBUG [Thread-2422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:14,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:14,960 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:14,960 DEBUG [Thread-2433 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:26:14,963 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:26:14,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:14,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:26:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:26:14,972 DEBUG [Thread-2418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:14,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:14,980 DEBUG [Thread-2431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:14,981 DEBUG [Thread-2431 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:14,981 DEBUG [Thread-2431 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:14,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T18:26:14,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:14,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:14,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:26:14,984 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:26:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:26:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2024-11-13T18:26:14,986 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:26:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-13T18:26:14,986 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:26:14,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:26:14,998 DEBUG [Thread-2427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:15,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:26:15,014 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:15,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:26:15,040 DEBUG [Thread-2414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:15,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:15,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522375023, max timestamp=1731522375041Num uniques 859; 2024-11-13T18:26:15,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:26:15,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522375023, max timestamp=1731522375041Num uniques 859; 2024-11-13T18:26:15,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:26:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:26:15,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522375023, max timestamp=1731522375041Num uniques 859; 2024-11-13T18:26:15,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:26:15,055 DEBUG [Thread-2425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:26:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=62ae331128c9,39615,1731522225114, seqNum=-1] 2024-11-13T18:26:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:26:15,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:26:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5] 2024-11-13T18:26:15,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132d35ed8e39db4940831973bbee533fa7_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522374976/Put/seqid=0 2024-11-13T18:26:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-13T18:26:15,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522435099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522435100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731522375076, max timestamp=1731522375106Num uniques 1018; 2024-11-13T18:26:15,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-13T18:26:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742366_1542 (size=12154) 2024-11-13T18:26:15,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522375076, max timestamp=1731522375106Num uniques 1009; 2024-11-13T18:26:15,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-13T18:26:15,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522375076, max timestamp=1731522375106Num uniques 1018; 2024-11-13T18:26:15,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-13T18:26:15,120 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:15,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,159 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132d35ed8e39db4940831973bbee533fa7_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132d35ed8e39db4940831973bbee533fa7_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:15,162 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/c819c406fadd4ec6bf115a456aa66961, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:15,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/c819c406fadd4ec6bf115a456aa66961 is 175, key is test_row_0/A:col10/1731522374976/Put/seqid=0 2024-11-13T18:26:15,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742367_1543 (size=30955) 2024-11-13T18:26:15,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522435213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522435216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522435217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522435220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731522375207, max timestamp=1731522375218Num uniques 1127; 2024-11-13T18:26:15,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731522375207, max timestamp=1731522375218Num uniques 1009; 2024-11-13T18:26:15,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-13T18:26:15,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-13T18:26:15,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731522375207, max timestamp=1731522375218Num uniques 1009; 2024-11-13T18:26:15,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-13T18:26:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-13T18:26:15,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522435334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522435336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522435337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522435426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522435436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522435445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522435456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:15,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522435556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,607 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/c819c406fadd4ec6bf115a456aa66961 2024-11-13T18:26:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-13T18:26:15,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:15,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:15,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/cefa50878a424774bf64903f20259523 is 50, key is test_row_0/B:col10/1731522374976/Put/seqid=0 2024-11-13T18:26:15,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522435658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522435666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742368_1544 (size=12001) 2024-11-13T18:26:15,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/cefa50878a424774bf64903f20259523 2024-11-13T18:26:15,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/cc8ae2bd1a3445bca9da73d836ec3a9b is 50, key is test_row_0/C:col10/1731522374976/Put/seqid=0 2024-11-13T18:26:15,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522435754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522435756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:15,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742369_1545 (size=12001) 2024-11-13T18:26:15,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/cc8ae2bd1a3445bca9da73d836ec3a9b 2024-11-13T18:26:15,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/c819c406fadd4ec6bf115a456aa66961 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/c819c406fadd4ec6bf115a456aa66961 2024-11-13T18:26:15,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/c819c406fadd4ec6bf115a456aa66961, entries=150, sequenceid=14, filesize=30.2 K 2024-11-13T18:26:15,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/cefa50878a424774bf64903f20259523 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/cefa50878a424774bf64903f20259523 2024-11-13T18:26:15,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/cefa50878a424774bf64903f20259523, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T18:26:15,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/cc8ae2bd1a3445bca9da73d836ec3a9b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cc8ae2bd1a3445bca9da73d836ec3a9b 2024-11-13T18:26:15,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cc8ae2bd1a3445bca9da73d836ec3a9b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T18:26:15,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2bf982671ac78b0bc57c6ef3a6b571eb in 874ms, sequenceid=14, compaction requested=false 2024-11-13T18:26:15,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:15,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731522375290, max timestamp=1731522375872Num uniques 1036; 2024-11-13T18:26:15,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-13T18:26:15,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731522375290, max timestamp=1731522375872Num uniques 1036; 2024-11-13T18:26:15,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-13T18:26:15,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731522375290, max timestamp=1731522375872Num uniques 1036; 2024-11-13T18:26:15,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-13T18:26:15,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=53.67 KB heapSize=143.39 KB 2024-11-13T18:26:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:26:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:26:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:26:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:15,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:15,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:15,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:15,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:15,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b6e36e8486f648bdb970fc98570bf79a_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522375882/Put/seqid=0 2024-11-13T18:26:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742370_1546 (size=12154) 2024-11-13T18:26:16,005 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:16,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522375978, max timestamp=1731522376011Num uniques 859; 2024-11-13T18:26:16,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:26:16,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731522375978, max timestamp=1731522376011Num uniques 859; 2024-11-13T18:26:16,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T18:26:16,017 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b6e36e8486f648bdb970fc98570bf79a_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b6e36e8486f648bdb970fc98570bf79a_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:16,018 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/6a9ddfcd199b4c6db16fe8c597d29720, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:16,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/6a9ddfcd199b4c6db16fe8c597d29720 is 175, key is test_row_0/A:col10/1731522375882/Put/seqid=0 2024-11-13T18:26:16,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731522375978, max timestamp=1731522376011Num uniques 859; 2024-11-13T18:26:16,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T18:26:16,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742371_1547 (size=30955) 2024-11-13T18:26:16,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/6a9ddfcd199b4c6db16fe8c597d29720 2024-11-13T18:26:16,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522436069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522436078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522436083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/785275aaad8640dfa931eb95d3692237 is 50, key is test_row_0/B:col10/1731522375882/Put/seqid=0 2024-11-13T18:26:16,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:16,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:16,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-13T18:26:16,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742372_1548 (size=12001) 2024-11-13T18:26:16,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/785275aaad8640dfa931eb95d3692237 2024-11-13T18:26:16,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522436185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/c70a58b6487e42feb1ae5d896139a955 is 50, key is test_row_0/C:col10/1731522375882/Put/seqid=0 2024-11-13T18:26:16,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522436200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522436201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742373_1549 (size=12001) 2024-11-13T18:26:16,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/c70a58b6487e42feb1ae5d896139a955 2024-11-13T18:26:16,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/6a9ddfcd199b4c6db16fe8c597d29720 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/6a9ddfcd199b4c6db16fe8c597d29720 2024-11-13T18:26:16,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/6a9ddfcd199b4c6db16fe8c597d29720, entries=150, sequenceid=90, filesize=30.2 K 2024-11-13T18:26:16,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/785275aaad8640dfa931eb95d3692237 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/785275aaad8640dfa931eb95d3692237 2024-11-13T18:26:16,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:16,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522436266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/785275aaad8640dfa931eb95d3692237, entries=150, sequenceid=90, filesize=11.7 K 2024-11-13T18:26:16,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522436276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/c70a58b6487e42feb1ae5d896139a955 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/c70a58b6487e42feb1ae5d896139a955 2024-11-13T18:26:16,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/c70a58b6487e42feb1ae5d896139a955, entries=150, sequenceid=90, filesize=11.7 K 2024-11-13T18:26:16,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~143.34 KB/146784, currentSize=133.78 KB/136995 for 2bf982671ac78b0bc57c6ef3a6b571eb in 389ms, sequenceid=90, compaction requested=false 2024-11-13T18:26:16,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:16,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=140.49 KB heapSize=404.70 KB 2024-11-13T18:26:16,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731522376066, max timestamp=1731522376399Num uniques 1018; 2024-11-13T18:26:16,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/62ae331128c9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T18:26:16,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:16,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T18:26:16,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:16,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:26:16,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:16,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T18:26:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:16,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:16,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:16,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a2ee873c96414e1793d4b3b5b9be53ee_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522376068/Put/seqid=0 2024-11-13T18:26:16,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742374_1550 (size=14594) 2024-11-13T18:26:16,472 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522436472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,508 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a2ee873c96414e1793d4b3b5b9be53ee_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a2ee873c96414e1793d4b3b5b9be53ee_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:16,509 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/159ae1028782407f98dc0f8457464fc6, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:16,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/159ae1028782407f98dc0f8457464fc6 is 175, key is test_row_0/A:col10/1731522376068/Put/seqid=0 2024-11-13T18:26:16,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522436485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522436488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742375_1551 (size=39549) 2024-11-13T18:26:16,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522436599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522436616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522436627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:16,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:16,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522436816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522436836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522436851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:16,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:16,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:16,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:16,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:16,966 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/159ae1028782407f98dc0f8457464fc6 2024-11-13T18:26:16,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/7abcae5c36c34ec0bbf9e0a2a939ed18 is 50, key is test_row_0/B:col10/1731522376068/Put/seqid=0 2024-11-13T18:26:17,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742376_1552 (size=12001) 2024-11-13T18:26:17,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/7abcae5c36c34ec0bbf9e0a2a939ed18 2024-11-13T18:26:17,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/d2742ae5a6da44a58a42e65f6ee4baff is 50, key is test_row_0/C:col10/1731522376068/Put/seqid=0 2024-11-13T18:26:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-13T18:26:17,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522437135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:17,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522437146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:17,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742377_1553 (size=12001) 2024-11-13T18:26:17,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522437175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:17,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522437278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,281 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in 2bf982671ac78b0bc57c6ef3a6b571eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:17,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522437286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,294 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in 2bf982671ac78b0bc57c6ef3a6b571eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:17,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/d2742ae5a6da44a58a42e65f6ee4baff 2024-11-13T18:26:17,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/159ae1028782407f98dc0f8457464fc6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/159ae1028782407f98dc0f8457464fc6 2024-11-13T18:26:17,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/159ae1028782407f98dc0f8457464fc6, entries=200, sequenceid=128, filesize=38.6 K 2024-11-13T18:26:17,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/7abcae5c36c34ec0bbf9e0a2a939ed18 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/7abcae5c36c34ec0bbf9e0a2a939ed18 2024-11-13T18:26:17,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/7abcae5c36c34ec0bbf9e0a2a939ed18, entries=150, sequenceid=128, filesize=11.7 K 2024-11-13T18:26:17,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/d2742ae5a6da44a58a42e65f6ee4baff as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2742ae5a6da44a58a42e65f6ee4baff 2024-11-13T18:26:17,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2742ae5a6da44a58a42e65f6ee4baff, entries=150, sequenceid=128, filesize=11.7 K 2024-11-13T18:26:17,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.48 KB/104935, heapSize ~291.24 KB/298232, currentSize=80.51 KB/82440 for 2bf982671ac78b0bc57c6ef3a6b571eb in 1210ms, sequenceid=128, compaction requested=true 2024-11-13T18:26:17,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:17,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:17,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:17,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:17,613 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:17,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:17,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:17,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:26:17,614 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:17,627 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:17,627 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/B is initiating minor compaction (all files) 2024-11-13T18:26:17,627 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/B in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,627 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/cefa50878a424774bf64903f20259523, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/785275aaad8640dfa931eb95d3692237, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/7abcae5c36c34ec0bbf9e0a2a939ed18] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=35.2 K 2024-11-13T18:26:17,627 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:17,627 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/A is initiating minor compaction (all files) 2024-11-13T18:26:17,627 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/A in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,628 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/c819c406fadd4ec6bf115a456aa66961, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/6a9ddfcd199b4c6db16fe8c597d29720, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/159ae1028782407f98dc0f8457464fc6] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=99.1 K 2024-11-13T18:26:17,628 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,628 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/c819c406fadd4ec6bf115a456aa66961, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/6a9ddfcd199b4c6db16fe8c597d29720, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/159ae1028782407f98dc0f8457464fc6] 2024-11-13T18:26:17,628 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting c819c406fadd4ec6bf115a456aa66961, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731522374962 2024-11-13T18:26:17,628 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting cefa50878a424774bf64903f20259523, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731522374962 2024-11-13T18:26:17,629 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a9ddfcd199b4c6db16fe8c597d29720, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731522375882 2024-11-13T18:26:17,629 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 785275aaad8640dfa931eb95d3692237, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731522375882 2024-11-13T18:26:17,630 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 7abcae5c36c34ec0bbf9e0a2a939ed18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731522376066 2024-11-13T18:26:17,630 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 159ae1028782407f98dc0f8457464fc6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731522376066 2024-11-13T18:26:17,651 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#B#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:17,651 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/c40cffcebdaa4edd9b03068f4a9f3e05 is 50, key is test_row_0/B:col10/1731522376068/Put/seqid=0 2024-11-13T18:26:17,662 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:17,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:26:17,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:17,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:17,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:17,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:17,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:17,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:17,680 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113f1442f1acf9f48b3a5b32ba84005b6d8_2bf982671ac78b0bc57c6ef3a6b571eb store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:17,682 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113f1442f1acf9f48b3a5b32ba84005b6d8_2bf982671ac78b0bc57c6ef3a6b571eb, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:17,682 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f1442f1acf9f48b3a5b32ba84005b6d8_2bf982671ac78b0bc57c6ef3a6b571eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:17,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742378_1554 (size=12104) 2024-11-13T18:26:17,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742379_1555 (size=4469) 2024-11-13T18:26:17,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131c5f1d19c7f141c08260149a09ef894c_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522376461/Put/seqid=0 2024-11-13T18:26:17,742 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/c40cffcebdaa4edd9b03068f4a9f3e05 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/c40cffcebdaa4edd9b03068f4a9f3e05 2024-11-13T18:26:17,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522437734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:17,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522437736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:17,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522437744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,753 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/B of 2bf982671ac78b0bc57c6ef3a6b571eb into c40cffcebdaa4edd9b03068f4a9f3e05(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:17,753 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:17,753 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/B, priority=13, startTime=1731522377613; duration=0sec 2024-11-13T18:26:17,753 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:17,753 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:B 2024-11-13T18:26:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,754 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:17,756 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:17,756 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/C is initiating minor compaction (all files) 2024-11-13T18:26:17,756 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/C in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,756 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cc8ae2bd1a3445bca9da73d836ec3a9b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/c70a58b6487e42feb1ae5d896139a955, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2742ae5a6da44a58a42e65f6ee4baff] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=35.2 K 2024-11-13T18:26:17,756 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting cc8ae2bd1a3445bca9da73d836ec3a9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731522374962 2024-11-13T18:26:17,757 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c70a58b6487e42feb1ae5d896139a955, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731522375882 2024-11-13T18:26:17,757 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting d2742ae5a6da44a58a42e65f6ee4baff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731522376066 2024-11-13T18:26:17,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742380_1556 (size=14794) 2024-11-13T18:26:17,762 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:17,775 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131c5f1d19c7f141c08260149a09ef894c_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c5f1d19c7f141c08260149a09ef894c_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:17,778 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#C#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:17,778 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/691639da75d147ab88ad072a93f3c9f7 is 50, key is test_row_0/C:col10/1731522376068/Put/seqid=0 2024-11-13T18:26:17,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/ba81572796c94375b2150f36837cf706, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:17,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/ba81572796c94375b2150f36837cf706 is 175, key is test_row_0/A:col10/1731522376461/Put/seqid=0 2024-11-13T18:26:17,816 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:26:17,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742381_1557 (size=39749) 2024-11-13T18:26:17,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:17,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:17,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522437846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:17,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742382_1558 (size=12104) 2024-11-13T18:26:17,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522437856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:17,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522437856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:18,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:18,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:18,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522438057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:18,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522438077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:18,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522438077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:18,139 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#A#compaction#472 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:18,139 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/ca78f7600f1a4790857718597002a30f is 175, key is test_row_0/A:col10/1731522376068/Put/seqid=0 2024-11-13T18:26:18,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:18,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:18,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742383_1559 (size=31058) 2024-11-13T18:26:18,224 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/ba81572796c94375b2150f36837cf706 2024-11-13T18:26:18,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/3a777837916440c99b9d4ec10846f76a is 50, key is test_row_0/B:col10/1731522376461/Put/seqid=0 2024-11-13T18:26:18,271 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/691639da75d147ab88ad072a93f3c9f7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/691639da75d147ab88ad072a93f3c9f7 2024-11-13T18:26:18,278 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/C of 2bf982671ac78b0bc57c6ef3a6b571eb into 691639da75d147ab88ad072a93f3c9f7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:18,278 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:18,278 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/C, priority=13, startTime=1731522377613; duration=0sec 2024-11-13T18:26:18,279 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:18,279 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:C 2024-11-13T18:26:18,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:18,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742384_1560 (size=12151) 2024-11-13T18:26:18,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/3a777837916440c99b9d4ec10846f76a 2024-11-13T18:26:18,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/afa84e0279c34ecaba0443dc7776603b is 50, key is test_row_0/C:col10/1731522376461/Put/seqid=0 2024-11-13T18:26:18,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522438376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:18,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522438385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:18,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522438396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:18,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742385_1561 (size=12151) 2024-11-13T18:26:18,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,588 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/ca78f7600f1a4790857718597002a30f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ca78f7600f1a4790857718597002a30f 2024-11-13T18:26:18,602 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/A of 2bf982671ac78b0bc57c6ef3a6b571eb into ca78f7600f1a4790857718597002a30f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:18,602 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:18,602 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/A, priority=13, startTime=1731522377613; duration=0sec 2024-11-13T18:26:18,602 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:18,602 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:A 2024-11-13T18:26:18,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/afa84e0279c34ecaba0443dc7776603b 2024-11-13T18:26:18,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:18,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/ba81572796c94375b2150f36837cf706 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ba81572796c94375b2150f36837cf706 2024-11-13T18:26:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ba81572796c94375b2150f36837cf706, entries=200, sequenceid=144, filesize=38.8 K 2024-11-13T18:26:18,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/3a777837916440c99b9d4ec10846f76a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3a777837916440c99b9d4ec10846f76a 2024-11-13T18:26:18,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3a777837916440c99b9d4ec10846f76a, entries=150, sequenceid=144, filesize=11.9 K 2024-11-13T18:26:18,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/afa84e0279c34ecaba0443dc7776603b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/afa84e0279c34ecaba0443dc7776603b 2024-11-13T18:26:18,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/afa84e0279c34ecaba0443dc7776603b, entries=150, sequenceid=144, filesize=11.9 K 2024-11-13T18:26:18,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2bf982671ac78b0bc57c6ef3a6b571eb in 1191ms, sequenceid=144, compaction requested=false 2024-11-13T18:26:18,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:18,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T18:26:18,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:18,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:18,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136ea5da40c95043ea84a7ff77ef935c5a_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522377732/Put/seqid=0 2024-11-13T18:26:18,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742386_1562 (size=14794) 2024-11-13T18:26:18,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:18,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522438964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:18,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522438965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:18,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:18,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:18,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:18,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522438959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522439082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522439082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522439099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:19,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-13T18:26:19,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522439295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522439295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522439296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,307 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in 2bf982671ac78b0bc57c6ef3a6b571eb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522439306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,314 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in 2bf982671ac78b0bc57c6ef3a6b571eb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:19,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522439315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,341 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,358 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136ea5da40c95043ea84a7ff77ef935c5a_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136ea5da40c95043ea84a7ff77ef935c5a_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:19,365 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/d420e67dd878462d8ac497ed9806a690, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:19,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/d420e67dd878462d8ac497ed9806a690 is 175, key is test_row_0/A:col10/1731522377732/Put/seqid=0 2024-11-13T18:26:19,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742387_1563 (size=39749) 2024-11-13T18:26:19,411 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/d420e67dd878462d8ac497ed9806a690 2024-11-13T18:26:19,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/5e157d97752d493494695512051b2152 is 50, key is test_row_0/B:col10/1731522377732/Put/seqid=0 2024-11-13T18:26:19,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742388_1564 (size=12151) 2024-11-13T18:26:19,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:19,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:19,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522439616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522439618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522439648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:19,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:19,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:19,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:19,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:19,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/5e157d97752d493494695512051b2152 2024-11-13T18:26:19,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/928f3aab379b434cb5afa582987eb57d is 50, key is test_row_0/C:col10/1731522377732/Put/seqid=0 2024-11-13T18:26:20,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742389_1565 (size=12151) 2024-11-13T18:26:20,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:20,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:20,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522440136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:20,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:20,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:20,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:20,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522440139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:20,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:20,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522440207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:20,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:20,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:20,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/928f3aab379b434cb5afa582987eb57d 2024-11-13T18:26:20,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/d420e67dd878462d8ac497ed9806a690 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d420e67dd878462d8ac497ed9806a690 2024-11-13T18:26:20,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d420e67dd878462d8ac497ed9806a690, entries=200, sequenceid=164, filesize=38.8 K 2024-11-13T18:26:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/5e157d97752d493494695512051b2152 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/5e157d97752d493494695512051b2152 2024-11-13T18:26:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/5e157d97752d493494695512051b2152, entries=150, sequenceid=164, filesize=11.9 K 2024-11-13T18:26:20,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/928f3aab379b434cb5afa582987eb57d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/928f3aab379b434cb5afa582987eb57d 2024-11-13T18:26:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:20,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:20,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/928f3aab379b434cb5afa582987eb57d, entries=150, sequenceid=164, filesize=11.9 K 2024-11-13T18:26:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 2bf982671ac78b0bc57c6ef3a6b571eb in 1635ms, sequenceid=164, compaction requested=true 2024-11-13T18:26:20,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:20,523 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:20,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:20,524 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:20,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:20,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:20,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:20,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:20,527 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:20,527 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/A is initiating minor compaction (all files) 2024-11-13T18:26:20,527 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/A in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,528 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ca78f7600f1a4790857718597002a30f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ba81572796c94375b2150f36837cf706, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d420e67dd878462d8ac497ed9806a690] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=108.0 K 2024-11-13T18:26:20,528 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,528 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ca78f7600f1a4790857718597002a30f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ba81572796c94375b2150f36837cf706, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d420e67dd878462d8ac497ed9806a690] 2024-11-13T18:26:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,530 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:20,530 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/B is initiating minor compaction (all files) 2024-11-13T18:26:20,530 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/B in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,530 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/c40cffcebdaa4edd9b03068f4a9f3e05, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3a777837916440c99b9d4ec10846f76a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/5e157d97752d493494695512051b2152] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=35.6 K 2024-11-13T18:26:20,531 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting c40cffcebdaa4edd9b03068f4a9f3e05, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731522376066 2024-11-13T18:26:20,531 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca78f7600f1a4790857718597002a30f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731522376066 2024-11-13T18:26:20,535 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba81572796c94375b2150f36837cf706, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731522376461 2024-11-13T18:26:20,535 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a777837916440c99b9d4ec10846f76a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731522376461 2024-11-13T18:26:20,535 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting d420e67dd878462d8ac497ed9806a690, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731522377732 2024-11-13T18:26:20,535 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e157d97752d493494695512051b2152, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731522377732 2024-11-13T18:26:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,573 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#B#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:20,573 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/b4da19b6232941d5ac0e6ed648f9dcb5 is 50, key is test_row_0/B:col10/1731522377732/Put/seqid=0 2024-11-13T18:26:20,574 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,588 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411130f591ba84fb14b14a56c21848c624340_2bf982671ac78b0bc57c6ef3a6b571eb store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,590 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411130f591ba84fb14b14a56c21848c624340_2bf982671ac78b0bc57c6ef3a6b571eb, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,590 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130f591ba84fb14b14a56c21848c624340_2bf982671ac78b0bc57c6ef3a6b571eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-13T18:26:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,655 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:26:20,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:20,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:20,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:20,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:20,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:20,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742391_1567 (size=4469) 2024-11-13T18:26:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742390_1566 (size=12357) 2024-11-13T18:26:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,674 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#A#compaction#481 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:20,675 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/f3506b6a4cf1445b809137f9890f375b is 175, key is test_row_0/A:col10/1731522377732/Put/seqid=0 2024-11-13T18:26:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ee25f6dd2e554aa0be4aef3530221aa0_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522378963/Put/seqid=0 2024-11-13T18:26:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,724 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/b4da19b6232941d5ac0e6ed648f9dcb5 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/b4da19b6232941d5ac0e6ed648f9dcb5 2024-11-13T18:26:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742392_1568 (size=31311) 2024-11-13T18:26:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,767 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/B of 2bf982671ac78b0bc57c6ef3a6b571eb into b4da19b6232941d5ac0e6ed648f9dcb5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:20,767 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:20,767 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/B, priority=13, startTime=1731522380524; duration=0sec 2024-11-13T18:26:20,767 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:20,767 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:B 2024-11-13T18:26:20,767 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,771 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:20,771 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/C is initiating minor compaction (all files) 2024-11-13T18:26:20,771 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/C in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:20,771 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/691639da75d147ab88ad072a93f3c9f7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/afa84e0279c34ecaba0443dc7776603b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/928f3aab379b434cb5afa582987eb57d] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=35.6 K 2024-11-13T18:26:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,774 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 691639da75d147ab88ad072a93f3c9f7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731522376066 2024-11-13T18:26:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,774 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting afa84e0279c34ecaba0443dc7776603b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731522376461 2024-11-13T18:26:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,775 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 928f3aab379b434cb5afa582987eb57d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731522377732 2024-11-13T18:26:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742393_1569 (size=9814) 2024-11-13T18:26:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,802 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#C#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,802 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/911083143bcd43ea901914847ddd864e is 50, key is test_row_0/C:col10/1731522377732/Put/seqid=0 2024-11-13T18:26:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742394_1570 (size=12357) 2024-11-13T18:26:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,877 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/911083143bcd43ea901914847ddd864e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/911083143bcd43ea901914847ddd864e 2024-11-13T18:26:20,883 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/C of 2bf982671ac78b0bc57c6ef3a6b571eb into 911083143bcd43ea901914847ddd864e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:20,883 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:20,883 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/C, priority=13, startTime=1731522380525; duration=0sec 2024-11-13T18:26:20,883 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:20,883 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:C 2024-11-13T18:26:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,163 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/f3506b6a4cf1445b809137f9890f375b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/f3506b6a4cf1445b809137f9890f375b 2024-11-13T18:26:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,169 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/A of 2bf982671ac78b0bc57c6ef3a6b571eb into f3506b6a4cf1445b809137f9890f375b(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:21,169 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,169 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/A, priority=13, startTime=1731522380523; duration=0sec 2024-11-13T18:26:21,170 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:21,170 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:A 2024-11-13T18:26:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,197 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ee25f6dd2e554aa0be4aef3530221aa0_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ee25f6dd2e554aa0be4aef3530221aa0_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/09400d81534340be9d87487fa8dccb54, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:21,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/09400d81534340be9d87487fa8dccb54 is 175, key is test_row_0/A:col10/1731522378963/Put/seqid=0 2024-11-13T18:26:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742395_1571 (size=22461) 2024-11-13T18:26:21,208 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/09400d81534340be9d87487fa8dccb54 2024-11-13T18:26:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:21,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/9a54051ff0c641659970d683ad988022 is 50, key is test_row_0/B:col10/1731522378963/Put/seqid=0 2024-11-13T18:26:21,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522441255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:21,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522441255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:21,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522441263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:21,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742396_1572 (size=9757) 2024-11-13T18:26:21,308 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/9a54051ff0c641659970d683ad988022 2024-11-13T18:26:21,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/1b1c59c55f04428088a3a8e1aa33bb73 is 50, key is test_row_0/C:col10/1731522378963/Put/seqid=0 2024-11-13T18:26:21,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522441376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:21,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522441376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:21,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742397_1573 (size=9757) 2024-11-13T18:26:21,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522441386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:21,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522441599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:21,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522441599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:21,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522441606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:21,788 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/1b1c59c55f04428088a3a8e1aa33bb73 2024-11-13T18:26:21,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/09400d81534340be9d87487fa8dccb54 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/09400d81534340be9d87487fa8dccb54 2024-11-13T18:26:21,821 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/09400d81534340be9d87487fa8dccb54, entries=100, sequenceid=180, filesize=21.9 K 2024-11-13T18:26:21,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/9a54051ff0c641659970d683ad988022 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a54051ff0c641659970d683ad988022 2024-11-13T18:26:21,830 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a54051ff0c641659970d683ad988022, entries=100, sequenceid=180, filesize=9.5 K 2024-11-13T18:26:21,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/1b1c59c55f04428088a3a8e1aa33bb73 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1b1c59c55f04428088a3a8e1aa33bb73 2024-11-13T18:26:21,848 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1b1c59c55f04428088a3a8e1aa33bb73, entries=100, sequenceid=180, filesize=9.5 K 2024-11-13T18:26:21,857 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2bf982671ac78b0bc57c6ef3a6b571eb in 1202ms, sequenceid=180, compaction requested=false 2024-11-13T18:26:21,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:21,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:21,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=135 2024-11-13T18:26:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=135 2024-11-13T18:26:21,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-13T18:26:21,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 6.8730 sec 2024-11-13T18:26:21,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 6.8900 sec 2024-11-13T18:26:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:21,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:26:21,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:21,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:21,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:21,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:21,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:21,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:21,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132c2f2c80e04b47b78d23c1bbdb446112_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522381924/Put/seqid=0 2024-11-13T18:26:22,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742398_1574 (size=14794) 2024-11-13T18:26:22,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:22,024 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132c2f2c80e04b47b78d23c1bbdb446112_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c2f2c80e04b47b78d23c1bbdb446112_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:22,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/5b26704d94c249e8bac935d3edd4f9d3, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:22,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/5b26704d94c249e8bac935d3edd4f9d3 is 175, key is test_row_0/A:col10/1731522381924/Put/seqid=0 2024-11-13T18:26:22,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522442017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522442029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522442029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742399_1575 (size=39749) 2024-11-13T18:26:22,070 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/5b26704d94c249e8bac935d3edd4f9d3 2024-11-13T18:26:22,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/79e64f779fbe46ac8ef5381a2d83c47e is 50, key is test_row_0/B:col10/1731522381924/Put/seqid=0 2024-11-13T18:26:22,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742400_1576 (size=12151) 2024-11-13T18:26:22,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522442136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522442147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522442148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522442346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522442355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522442355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/79e64f779fbe46ac8ef5381a2d83c47e 2024-11-13T18:26:22,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/85bc0588ac79466b86ff5862bec8a7e6 is 50, key is test_row_0/C:col10/1731522381924/Put/seqid=0 2024-11-13T18:26:22,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742401_1577 (size=12151) 2024-11-13T18:26:22,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522442666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522442668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522442689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:22,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/85bc0588ac79466b86ff5862bec8a7e6 2024-11-13T18:26:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/5b26704d94c249e8bac935d3edd4f9d3 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/5b26704d94c249e8bac935d3edd4f9d3 2024-11-13T18:26:23,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/5b26704d94c249e8bac935d3edd4f9d3, entries=200, sequenceid=202, filesize=38.8 K 2024-11-13T18:26:23,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/79e64f779fbe46ac8ef5381a2d83c47e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/79e64f779fbe46ac8ef5381a2d83c47e 2024-11-13T18:26:23,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/79e64f779fbe46ac8ef5381a2d83c47e, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T18:26:23,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/85bc0588ac79466b86ff5862bec8a7e6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/85bc0588ac79466b86ff5862bec8a7e6 2024-11-13T18:26:23,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/85bc0588ac79466b86ff5862bec8a7e6, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T18:26:23,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 2bf982671ac78b0bc57c6ef3a6b571eb in 1146ms, sequenceid=202, compaction requested=true 2024-11-13T18:26:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:23,075 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:23,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:23,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:23,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:23,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:23,076 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:23,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:23,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:23,091 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:23,091 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/A is initiating minor compaction (all files) 2024-11-13T18:26:23,091 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/A in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,091 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/f3506b6a4cf1445b809137f9890f375b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/09400d81534340be9d87487fa8dccb54, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/5b26704d94c249e8bac935d3edd4f9d3] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=91.3 K 2024-11-13T18:26:23,091 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,091 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/f3506b6a4cf1445b809137f9890f375b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/09400d81534340be9d87487fa8dccb54, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/5b26704d94c249e8bac935d3edd4f9d3] 2024-11-13T18:26:23,094 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:23,095 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/B is initiating minor compaction (all files) 2024-11-13T18:26:23,095 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/B in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,095 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/b4da19b6232941d5ac0e6ed648f9dcb5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a54051ff0c641659970d683ad988022, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/79e64f779fbe46ac8ef5381a2d83c47e] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=33.5 K 2024-11-13T18:26:23,095 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3506b6a4cf1445b809137f9890f375b, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731522377732 2024-11-13T18:26:23,098 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting b4da19b6232941d5ac0e6ed648f9dcb5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731522377732 2024-11-13T18:26:23,100 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09400d81534340be9d87487fa8dccb54, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731522378954 2024-11-13T18:26:23,100 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a54051ff0c641659970d683ad988022, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731522378954 2024-11-13T18:26:23,107 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b26704d94c249e8bac935d3edd4f9d3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522381244 2024-11-13T18:26:23,111 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e64f779fbe46ac8ef5381a2d83c47e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522381260 2024-11-13T18:26:23,158 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:23,159 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/8e98dc3b46194d469647a900f1cf3ca0 is 50, key is test_row_0/B:col10/1731522381924/Put/seqid=0 2024-11-13T18:26:23,160 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:23,180 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111316be2515b3c34b7fbc73de3c2e4ca4e8_2bf982671ac78b0bc57c6ef3a6b571eb store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:23,182 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111316be2515b3c34b7fbc73de3c2e4ca4e8_2bf982671ac78b0bc57c6ef3a6b571eb, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:23,182 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111316be2515b3c34b7fbc73de3c2e4ca4e8_2bf982671ac78b0bc57c6ef3a6b571eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:23,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:26:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-13T18:26:23,201 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:23,202 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:26:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-11-13T18:26:23,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742402_1578 (size=12459) 2024-11-13T18:26:23,205 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:26:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T18:26:23,207 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:26:23,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:26:23,227 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/8e98dc3b46194d469647a900f1cf3ca0 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8e98dc3b46194d469647a900f1cf3ca0 2024-11-13T18:26:23,234 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/B of 2bf982671ac78b0bc57c6ef3a6b571eb into 8e98dc3b46194d469647a900f1cf3ca0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:23,234 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:23,234 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/B, priority=13, startTime=1731522383076; duration=0sec 2024-11-13T18:26:23,235 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:23,235 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:B 2024-11-13T18:26:23,235 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:23,236 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:23,236 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/C is initiating minor compaction (all files) 2024-11-13T18:26:23,236 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/C in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,237 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/911083143bcd43ea901914847ddd864e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1b1c59c55f04428088a3a8e1aa33bb73, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/85bc0588ac79466b86ff5862bec8a7e6] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=33.5 K 2024-11-13T18:26:23,237 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 911083143bcd43ea901914847ddd864e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731522377732 2024-11-13T18:26:23,237 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b1c59c55f04428088a3a8e1aa33bb73, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731522378954 2024-11-13T18:26:23,238 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 85bc0588ac79466b86ff5862bec8a7e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522381260 2024-11-13T18:26:23,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742403_1579 (size=4469) 2024-11-13T18:26:23,244 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#A#compaction#490 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:23,244 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/b817aecd397e4c24ba3f86c1983174b7 is 175, key is test_row_0/A:col10/1731522381924/Put/seqid=0 2024-11-13T18:26:23,253 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:23,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111324e047ae96f94cd28cf017390b212bd6_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522383194/Put/seqid=0 2024-11-13T18:26:23,254 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/da9921f948b945df9d20f2c8001ff5d7 is 50, key is test_row_0/C:col10/1731522381924/Put/seqid=0 2024-11-13T18:26:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522443257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522443263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522443265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742404_1580 (size=31413) 2024-11-13T18:26:23,308 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/b817aecd397e4c24ba3f86c1983174b7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/b817aecd397e4c24ba3f86c1983174b7 2024-11-13T18:26:23,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742406_1582 (size=12459) 2024-11-13T18:26:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T18:26:23,323 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/A of 2bf982671ac78b0bc57c6ef3a6b571eb into b817aecd397e4c24ba3f86c1983174b7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:23,323 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:23,324 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/A, priority=13, startTime=1731522383075; duration=0sec 2024-11-13T18:26:23,324 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:23,324 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:A 2024-11-13T18:26:23,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742405_1581 (size=14794) 2024-11-13T18:26:23,329 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/da9921f948b945df9d20f2c8001ff5d7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/da9921f948b945df9d20f2c8001ff5d7 2024-11-13T18:26:23,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522443326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,340 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111324e047ae96f94cd28cf017390b212bd6_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111324e047ae96f94cd28cf017390b212bd6_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:23,342 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/C of 2bf982671ac78b0bc57c6ef3a6b571eb into da9921f948b945df9d20f2c8001ff5d7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:23,342 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:23,342 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/C, priority=13, startTime=1731522383076; duration=0sec 2024-11-13T18:26:23,342 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:23,342 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:C 2024-11-13T18:26:23,342 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_0' in 2bf982671ac78b0bc57c6ef3a6b571eb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:23,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,343 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/d068f78d49ab4dc6850931af927bfde8, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:23,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/d068f78d49ab4dc6850931af927bfde8 is 175, key is test_row_0/A:col10/1731522383194/Put/seqid=0 2024-11-13T18:26:23,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522443336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,346 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in 2bf982671ac78b0bc57c6ef3a6b571eb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742407_1583 (size=39749) 2024-11-13T18:26:23,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,383 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/d068f78d49ab4dc6850931af927bfde8 2024-11-13T18:26:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522443375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522443375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522443376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/9a10ea47a69b426a964f68f530312af6 is 50, key is test_row_0/B:col10/1731522383194/Put/seqid=0 2024-11-13T18:26:23,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742408_1584 (size=12151) 2024-11-13T18:26:23,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/9a10ea47a69b426a964f68f530312af6 2024-11-13T18:26:23,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/ff03213ff63c4415b04079c67075e8b9 is 50, key is test_row_0/C:col10/1731522383194/Put/seqid=0 2024-11-13T18:26:23,478 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:26:23,479 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T18:26:23,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742409_1585 (size=12151) 2024-11-13T18:26:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T18:26:23,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:23,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:23,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/ff03213ff63c4415b04079c67075e8b9 2024-11-13T18:26:23,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/d068f78d49ab4dc6850931af927bfde8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d068f78d49ab4dc6850931af927bfde8 2024-11-13T18:26:23,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d068f78d49ab4dc6850931af927bfde8, entries=200, sequenceid=217, filesize=38.8 K 2024-11-13T18:26:23,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/9a10ea47a69b426a964f68f530312af6 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a10ea47a69b426a964f68f530312af6 2024-11-13T18:26:23,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a10ea47a69b426a964f68f530312af6, entries=150, sequenceid=217, filesize=11.9 K 2024-11-13T18:26:23,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/ff03213ff63c4415b04079c67075e8b9 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ff03213ff63c4415b04079c67075e8b9 2024-11-13T18:26:23,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ff03213ff63c4415b04079c67075e8b9, entries=150, sequenceid=217, filesize=11.9 K 2024-11-13T18:26:23,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2bf982671ac78b0bc57c6ef3a6b571eb in 393ms, sequenceid=217, compaction requested=false 2024-11-13T18:26:23,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:23,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:26:23,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:23,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:23,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:23,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:23,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:23,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:23,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ab2a7e975ff64f738bde9de966e1a09a_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522383597/Put/seqid=0 2024-11-13T18:26:23,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742410_1586 (size=14794) 2024-11-13T18:26:23,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522443644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,652 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522443647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,661 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ab2a7e975ff64f738bde9de966e1a09a_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ab2a7e975ff64f738bde9de966e1a09a_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522443650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,674 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/07769939c187457bad5b2a8760e7ceec, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:23,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/07769939c187457bad5b2a8760e7ceec is 175, key is test_row_0/A:col10/1731522383597/Put/seqid=0 2024-11-13T18:26:23,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:23,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:23,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742411_1587 (size=39749) 2024-11-13T18:26:23,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/07769939c187457bad5b2a8760e7ceec 2024-11-13T18:26:23,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/6776805ee3314291ae15ee9a31240913 is 50, key is test_row_0/B:col10/1731522383597/Put/seqid=0 2024-11-13T18:26:23,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522443758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522443765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742412_1588 (size=12151) 2024-11-13T18:26:23,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/6776805ee3314291ae15ee9a31240913 2024-11-13T18:26:23,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522443776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:23,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/78cb2d1b555c4aad81922b0fdf31ae72 is 50, key is test_row_0/C:col10/1731522383597/Put/seqid=0 2024-11-13T18:26:23,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742413_1589 (size=12151) 2024-11-13T18:26:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T18:26:23,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/78cb2d1b555c4aad81922b0fdf31ae72 2024-11-13T18:26:23,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:23,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:23,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/07769939c187457bad5b2a8760e7ceec as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/07769939c187457bad5b2a8760e7ceec 2024-11-13T18:26:23,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:23,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/07769939c187457bad5b2a8760e7ceec, entries=200, sequenceid=238, filesize=38.8 K 2024-11-13T18:26:23,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/6776805ee3314291ae15ee9a31240913 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/6776805ee3314291ae15ee9a31240913 2024-11-13T18:26:23,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/6776805ee3314291ae15ee9a31240913, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T18:26:23,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/78cb2d1b555c4aad81922b0fdf31ae72 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/78cb2d1b555c4aad81922b0fdf31ae72 2024-11-13T18:26:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/78cb2d1b555c4aad81922b0fdf31ae72, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T18:26:23,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2bf982671ac78b0bc57c6ef3a6b571eb in 290ms, sequenceid=238, compaction requested=true 2024-11-13T18:26:23,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:23,891 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:26:23,892 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,903 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:23,903 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/A is initiating minor compaction (all files) 2024-11-13T18:26:23,904 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/A in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,904 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/b817aecd397e4c24ba3f86c1983174b7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d068f78d49ab4dc6850931af927bfde8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/07769939c187457bad5b2a8760e7ceec] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=108.3 K 2024-11-13T18:26:23,904 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,904 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/b817aecd397e4c24ba3f86c1983174b7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d068f78d49ab4dc6850931af927bfde8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/07769939c187457bad5b2a8760e7ceec] 2024-11-13T18:26:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,905 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:23,905 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/B is initiating minor compaction (all files) 2024-11-13T18:26:23,905 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/B in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,905 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8e98dc3b46194d469647a900f1cf3ca0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a10ea47a69b426a964f68f530312af6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/6776805ee3314291ae15ee9a31240913] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=35.9 K 2024-11-13T18:26:23,905 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting b817aecd397e4c24ba3f86c1983174b7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522381260 2024-11-13T18:26:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,906 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e98dc3b46194d469647a900f1cf3ca0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522381260 2024-11-13T18:26:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,907 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a10ea47a69b426a964f68f530312af6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731522381944 2024-11-13T18:26:23,908 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 6776805ee3314291ae15ee9a31240913, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731522383222 2024-11-13T18:26:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,908 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting d068f78d49ab4dc6850931af927bfde8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731522381944 2024-11-13T18:26:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,909 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07769939c187457bad5b2a8760e7ceec, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731522383222 2024-11-13T18:26:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,920 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:23,921 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/4d680e86d0a74050b3528eed55807262 is 50, key is test_row_0/B:col10/1731522383597/Put/seqid=0 2024-11-13T18:26:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,928 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,951 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111377bbf79c42b24d1aa0d75a4a070112ae_2bf982671ac78b0bc57c6ef3a6b571eb store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,954 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111377bbf79c42b24d1aa0d75a4a070112ae_2bf982671ac78b0bc57c6ef3a6b571eb, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:23,954 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111377bbf79c42b24d1aa0d75a4a070112ae_2bf982671ac78b0bc57c6ef3a6b571eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742414_1590 (size=12561) 2024-11-13T18:26:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:23,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T18:26:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:23,981 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/4d680e86d0a74050b3528eed55807262 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4d680e86d0a74050b3528eed55807262 2024-11-13T18:26:23,987 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/B of 2bf982671ac78b0bc57c6ef3a6b571eb into 4d680e86d0a74050b3528eed55807262(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:23,987 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:23,987 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/B, priority=13, startTime=1731522383891; duration=0sec 2024-11-13T18:26:23,987 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:23,987 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:B 2024-11-13T18:26:23,988 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:23,989 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:23,989 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/C is initiating minor compaction (all files) 2024-11-13T18:26:23,989 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/C in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:23,989 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/da9921f948b945df9d20f2c8001ff5d7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ff03213ff63c4415b04079c67075e8b9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/78cb2d1b555c4aad81922b0fdf31ae72] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=35.9 K 2024-11-13T18:26:23,993 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting da9921f948b945df9d20f2c8001ff5d7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731522381260 2024-11-13T18:26:23,994 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting ff03213ff63c4415b04079c67075e8b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731522381944 2024-11-13T18:26:23,995 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 78cb2d1b555c4aad81922b0fdf31ae72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731522383222 2024-11-13T18:26:24,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134b36556aa20a419b81de0b77f3f0c019_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522383648/Put/seqid=0 2024-11-13T18:26:24,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:24,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:24,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742415_1591 (size=4469) 2024-11-13T18:26:24,011 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#A#compaction#499 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:24,011 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/e50f5bc8775e4bc3962b1cc4e518f856 is 175, key is test_row_0/A:col10/1731522383597/Put/seqid=0 2024-11-13T18:26:24,055 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#C#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:24,055 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/07803082d4984909800009b97e55715a is 50, key is test_row_0/C:col10/1731522383597/Put/seqid=0 2024-11-13T18:26:24,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742416_1592 (size=12304) 2024-11-13T18:26:24,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742417_1593 (size=31515) 2024-11-13T18:26:24,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522444077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522444092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522444092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742418_1594 (size=12561) 2024-11-13T18:26:24,128 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/07803082d4984909800009b97e55715a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/07803082d4984909800009b97e55715a 2024-11-13T18:26:24,134 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/C of 2bf982671ac78b0bc57c6ef3a6b571eb into 07803082d4984909800009b97e55715a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:24,134 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:24,134 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/C, priority=13, startTime=1731522383891; duration=0sec 2024-11-13T18:26:24,135 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:24,135 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:C 2024-11-13T18:26:24,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522444196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522444216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522444216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:24,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:24,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T18:26:24,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522444408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522444436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522444436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:24,465 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134b36556aa20a419b81de0b77f3f0c019_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134b36556aa20a419b81de0b77f3f0c019_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:24,466 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/3a8ed835199d4108a5ca8b6465a5acea, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:24,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/3a8ed835199d4108a5ca8b6465a5acea is 175, key is test_row_0/A:col10/1731522383648/Put/seqid=0 2024-11-13T18:26:24,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:24,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:24,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,487 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/e50f5bc8775e4bc3962b1cc4e518f856 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/e50f5bc8775e4bc3962b1cc4e518f856 2024-11-13T18:26:24,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742419_1595 (size=31105) 2024-11-13T18:26:24,497 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/A of 2bf982671ac78b0bc57c6ef3a6b571eb into e50f5bc8775e4bc3962b1cc4e518f856(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:24,497 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:24,497 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/A, priority=13, startTime=1731522383890; duration=0sec 2024-11-13T18:26:24,497 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:24,497 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:A 2024-11-13T18:26:24,498 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/3a8ed835199d4108a5ca8b6465a5acea 2024-11-13T18:26:24,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/90f08e5cdd7e46c0a51f14066d882f06 is 50, key is test_row_0/B:col10/1731522383648/Put/seqid=0 2024-11-13T18:26:24,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742420_1596 (size=12151) 2024-11-13T18:26:24,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/90f08e5cdd7e46c0a51f14066d882f06 2024-11-13T18:26:24,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/933529dd67db4f25ab16e816a7a70f3f is 50, key is test_row_0/C:col10/1731522383648/Put/seqid=0 2024-11-13T18:26:24,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742421_1597 (size=12151) 2024-11-13T18:26:24,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:24,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:24,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522444719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522444746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522444746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:24,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:24,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:24,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:24,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:25,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/933529dd67db4f25ab16e816a7a70f3f 2024-11-13T18:26:25,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/3a8ed835199d4108a5ca8b6465a5acea as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/3a8ed835199d4108a5ca8b6465a5acea 2024-11-13T18:26:25,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/3a8ed835199d4108a5ca8b6465a5acea, entries=150, sequenceid=253, filesize=30.4 K 2024-11-13T18:26:25,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/90f08e5cdd7e46c0a51f14066d882f06 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/90f08e5cdd7e46c0a51f14066d882f06 2024-11-13T18:26:25,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/90f08e5cdd7e46c0a51f14066d882f06, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T18:26:25,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/933529dd67db4f25ab16e816a7a70f3f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/933529dd67db4f25ab16e816a7a70f3f 2024-11-13T18:26:25,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/933529dd67db4f25ab16e816a7a70f3f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T18:26:25,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 2bf982671ac78b0bc57c6ef3a6b571eb in 1084ms, sequenceid=253, compaction requested=false 2024-11-13T18:26:25,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:25,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T18:26:25,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:25,098 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:26:25,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:25,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111368b57d3e462249cfb48ed703cbc689d3_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522384075/Put/seqid=0 2024-11-13T18:26:25,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742422_1598 (size=12454) 2024-11-13T18:26:25,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:25,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522445291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522445295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522445295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T18:26:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522445407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522445407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522445414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:25,585 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111368b57d3e462249cfb48ed703cbc689d3_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111368b57d3e462249cfb48ed703cbc689d3_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:25,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/2feb8f1552154c49bf95e1ecbbf1af88, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:25,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/2feb8f1552154c49bf95e1ecbbf1af88 is 175, key is test_row_0/A:col10/1731522384075/Put/seqid=0 2024-11-13T18:26:25,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742423_1599 (size=31255) 2024-11-13T18:26:25,618 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/2feb8f1552154c49bf95e1ecbbf1af88 2024-11-13T18:26:25,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522445627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522445627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522445631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/4b9f7fe442c8426a811e9551434c41b8 is 50, key is test_row_0/B:col10/1731522384075/Put/seqid=0 2024-11-13T18:26:25,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742424_1600 (size=12301) 2024-11-13T18:26:25,660 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/4b9f7fe442c8426a811e9551434c41b8 2024-11-13T18:26:25,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/1a8872294209442da51ccdba08679801 is 50, key is test_row_0/C:col10/1731522384075/Put/seqid=0 2024-11-13T18:26:25,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742425_1601 (size=12301) 2024-11-13T18:26:25,715 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/1a8872294209442da51ccdba08679801 2024-11-13T18:26:25,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/2feb8f1552154c49bf95e1ecbbf1af88 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/2feb8f1552154c49bf95e1ecbbf1af88 2024-11-13T18:26:25,738 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/2feb8f1552154c49bf95e1ecbbf1af88, entries=150, sequenceid=274, filesize=30.5 K 2024-11-13T18:26:25,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/4b9f7fe442c8426a811e9551434c41b8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4b9f7fe442c8426a811e9551434c41b8 2024-11-13T18:26:25,747 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4b9f7fe442c8426a811e9551434c41b8, entries=150, sequenceid=274, filesize=12.0 K 2024-11-13T18:26:25,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/1a8872294209442da51ccdba08679801 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1a8872294209442da51ccdba08679801 2024-11-13T18:26:25,756 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1a8872294209442da51ccdba08679801, entries=150, sequenceid=274, filesize=12.0 K 2024-11-13T18:26:25,757 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 2bf982671ac78b0bc57c6ef3a6b571eb in 659ms, sequenceid=274, compaction requested=true 2024-11-13T18:26:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=137 2024-11-13T18:26:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=137 2024-11-13T18:26:25,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-13T18:26:25,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5530 sec 2024-11-13T18:26:25,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 2.5680 sec 2024-11-13T18:26:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:25,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:26:25,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:25,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:25,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:25,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:25,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:25,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:25,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f6e03a0b81264260b71cbfe11c159451_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522385281/Put/seqid=0 2024-11-13T18:26:25,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522445971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522445974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:25,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522445983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:25,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:25,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:26,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742426_1602 (size=14994) 2024-11-13T18:26:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522446086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522446086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:26,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522446096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:26,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522446296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:26,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522446305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:26,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522446306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:26,403 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:26,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f6e03a0b81264260b71cbfe11c159451_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f6e03a0b81264260b71cbfe11c159451_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:26,409 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/bb8326853d6e47de83e10ef54da54bd7, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:26,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/bb8326853d6e47de83e10ef54da54bd7 is 175, key is test_row_0/A:col10/1731522385281/Put/seqid=0 2024-11-13T18:26:26,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742427_1603 (size=39949) 2024-11-13T18:26:26,455 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/bb8326853d6e47de83e10ef54da54bd7 2024-11-13T18:26:26,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/db6204de1f2d4c9cad23a88c401bca65 is 50, key is test_row_0/B:col10/1731522385281/Put/seqid=0 2024-11-13T18:26:26,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742428_1604 (size=12301) 2024-11-13T18:26:26,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522446616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:26,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:26,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522446616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:26,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:26,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522446617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:26,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:26,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:26,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/db6204de1f2d4c9cad23a88c401bca65 2024-11-13T18:26:26,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/cd8040353fc346dea33c0d705d0f2e2c is 50, key is test_row_0/C:col10/1731522385281/Put/seqid=0 2024-11-13T18:26:27,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742429_1605 (size=12301) 2024-11-13T18:26:27,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522447126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:27,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522447126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:27,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522447126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T18:26:27,396 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:27,397 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:26:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-11-13T18:26:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:26:27,399 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:26:27,407 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:26:27,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:26:27,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/cd8040353fc346dea33c0d705d0f2e2c 2024-11-13T18:26:27,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/bb8326853d6e47de83e10ef54da54bd7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/bb8326853d6e47de83e10ef54da54bd7 2024-11-13T18:26:27,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/bb8326853d6e47de83e10ef54da54bd7, entries=200, sequenceid=290, filesize=39.0 K 2024-11-13T18:26:27,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/db6204de1f2d4c9cad23a88c401bca65 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/db6204de1f2d4c9cad23a88c401bca65 2024-11-13T18:26:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:26:27,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/db6204de1f2d4c9cad23a88c401bca65, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T18:26:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/cd8040353fc346dea33c0d705d0f2e2c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cd8040353fc346dea33c0d705d0f2e2c 2024-11-13T18:26:27,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cd8040353fc346dea33c0d705d0f2e2c, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T18:26:27,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2bf982671ac78b0bc57c6ef3a6b571eb in 1614ms, sequenceid=290, compaction requested=true 2024-11-13T18:26:27,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:27,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:27,553 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:26:27,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:27,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:27,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:27,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:27,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:26:27,553 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:26:27,563 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49314 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:26:27,563 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/B is initiating minor compaction (all files) 2024-11-13T18:26:27,563 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/B in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:27,563 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4d680e86d0a74050b3528eed55807262, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/90f08e5cdd7e46c0a51f14066d882f06, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4b9f7fe442c8426a811e9551434c41b8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/db6204de1f2d4c9cad23a88c401bca65] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=48.2 K 2024-11-13T18:26:27,563 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133824 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:26:27,563 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/A is initiating minor compaction (all files) 2024-11-13T18:26:27,564 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/A in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:27,564 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/e50f5bc8775e4bc3962b1cc4e518f856, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/3a8ed835199d4108a5ca8b6465a5acea, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/2feb8f1552154c49bf95e1ecbbf1af88, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/bb8326853d6e47de83e10ef54da54bd7] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=130.7 K 2024-11-13T18:26:27,564 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:27,564 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/e50f5bc8775e4bc3962b1cc4e518f856, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/3a8ed835199d4108a5ca8b6465a5acea, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/2feb8f1552154c49bf95e1ecbbf1af88, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/bb8326853d6e47de83e10ef54da54bd7] 2024-11-13T18:26:27,564 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d680e86d0a74050b3528eed55807262, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731522383222 2024-11-13T18:26:27,565 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting e50f5bc8775e4bc3962b1cc4e518f856, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731522383222 2024-11-13T18:26:27,565 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 90f08e5cdd7e46c0a51f14066d882f06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731522383641 2024-11-13T18:26:27,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-13T18:26:27,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:27,566 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T18:26:27,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:27,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:27,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:27,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:27,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:27,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:27,571 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a8ed835199d4108a5ca8b6465a5acea, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731522383641 2024-11-13T18:26:27,571 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b9f7fe442c8426a811e9551434c41b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731522384075 2024-11-13T18:26:27,571 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2feb8f1552154c49bf95e1ecbbf1af88, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731522384075 2024-11-13T18:26:27,572 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting db6204de1f2d4c9cad23a88c401bca65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731522385276 2024-11-13T18:26:27,572 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb8326853d6e47de83e10ef54da54bd7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731522385276 2024-11-13T18:26:27,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111312962d76972c4790a6c0f71400c9fe27_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522385956/Put/seqid=0 2024-11-13T18:26:27,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742430_1606 (size=12454) 2024-11-13T18:26:27,650 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#B#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:27,650 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/3355b8190be94e71ae3bfaba1d4c2bf4 is 50, key is test_row_0/B:col10/1731522385281/Put/seqid=0 2024-11-13T18:26:27,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:27,660 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:27,679 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411130797ee8d839342ce9301a336e0ea0e98_2bf982671ac78b0bc57c6ef3a6b571eb store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:27,680 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111312962d76972c4790a6c0f71400c9fe27_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111312962d76972c4790a6c0f71400c9fe27_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:27,682 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411130797ee8d839342ce9301a336e0ea0e98_2bf982671ac78b0bc57c6ef3a6b571eb, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:27,683 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130797ee8d839342ce9301a336e0ea0e98_2bf982671ac78b0bc57c6ef3a6b571eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:27,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/aa144ee283b045c58f607bd289fb3ee1, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/aa144ee283b045c58f607bd289fb3ee1 is 175, key is test_row_0/A:col10/1731522385956/Put/seqid=0 2024-11-13T18:26:27,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742431_1607 (size=12847) 2024-11-13T18:26:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:26:27,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742432_1608 (size=4469) 2024-11-13T18:26:27,730 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#A#compaction#512 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:27,731 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/9bbe206a3b424d5d888408b20acb9e5f is 175, key is test_row_0/A:col10/1731522385281/Put/seqid=0 2024-11-13T18:26:27,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742433_1609 (size=31255) 2024-11-13T18:26:27,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742434_1610 (size=31801) 2024-11-13T18:26:27,795 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/9bbe206a3b424d5d888408b20acb9e5f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/9bbe206a3b424d5d888408b20acb9e5f 2024-11-13T18:26:27,839 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/A of 2bf982671ac78b0bc57c6ef3a6b571eb into 9bbe206a3b424d5d888408b20acb9e5f(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:27,839 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:27,839 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/A, priority=12, startTime=1731522387552; duration=0sec 2024-11-13T18:26:27,839 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:27,839 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:A 2024-11-13T18:26:27,840 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:26:27,851 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49314 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:26:27,851 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/C is initiating minor compaction (all files) 2024-11-13T18:26:27,851 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/C in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:27,851 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/07803082d4984909800009b97e55715a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/933529dd67db4f25ab16e816a7a70f3f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1a8872294209442da51ccdba08679801, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cd8040353fc346dea33c0d705d0f2e2c] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=48.2 K 2024-11-13T18:26:27,855 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07803082d4984909800009b97e55715a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731522383222 2024-11-13T18:26:27,855 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 933529dd67db4f25ab16e816a7a70f3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731522383641 2024-11-13T18:26:27,859 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a8872294209442da51ccdba08679801, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731522384075 2024-11-13T18:26:27,859 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd8040353fc346dea33c0d705d0f2e2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731522385276 2024-11-13T18:26:27,961 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#C#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:27,962 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/e15cce9ac1e54072a77be260e682c6c4 is 50, key is test_row_0/C:col10/1731522385281/Put/seqid=0 2024-11-13T18:26:28,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742435_1611 (size=12847) 2024-11-13T18:26:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:26:28,090 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/e15cce9ac1e54072a77be260e682c6c4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/e15cce9ac1e54072a77be260e682c6c4 2024-11-13T18:26:28,129 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/C of 2bf982671ac78b0bc57c6ef3a6b571eb into e15cce9ac1e54072a77be260e682c6c4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:28,129 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:28,129 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/C, priority=12, startTime=1731522387553; duration=0sec 2024-11-13T18:26:28,129 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:28,129 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:C 2024-11-13T18:26:28,143 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/3355b8190be94e71ae3bfaba1d4c2bf4 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3355b8190be94e71ae3bfaba1d4c2bf4 2024-11-13T18:26:28,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:28,155 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/B of 2bf982671ac78b0bc57c6ef3a6b571eb into 3355b8190be94e71ae3bfaba1d4c2bf4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:28,155 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:28,155 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/aa144ee283b045c58f607bd289fb3ee1 2024-11-13T18:26:28,155 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/B, priority=12, startTime=1731522387553; duration=0sec 2024-11-13T18:26:28,155 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:28,155 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:B 2024-11-13T18:26:28,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522448186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522448195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522448193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/351562bc1b764743888a9e1b81a0d016 is 50, key is test_row_0/B:col10/1731522385956/Put/seqid=0 2024-11-13T18:26:28,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742436_1612 (size=12301) 2024-11-13T18:26:28,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522448296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522448307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522448307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522448519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522448520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522448520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:26:28,671 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/351562bc1b764743888a9e1b81a0d016 2024-11-13T18:26:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/09936caa3c844ff38c534d70b2562fd8 is 50, key is test_row_0/C:col10/1731522385956/Put/seqid=0 2024-11-13T18:26:28,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742437_1613 (size=12301) 2024-11-13T18:26:28,755 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/09936caa3c844ff38c534d70b2562fd8 2024-11-13T18:26:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/aa144ee283b045c58f607bd289fb3ee1 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aa144ee283b045c58f607bd289fb3ee1 2024-11-13T18:26:28,820 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aa144ee283b045c58f607bd289fb3ee1, entries=150, sequenceid=306, filesize=30.5 K 2024-11-13T18:26:28,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/351562bc1b764743888a9e1b81a0d016 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/351562bc1b764743888a9e1b81a0d016 2024-11-13T18:26:28,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522448839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522448839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522448839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:28,864 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/351562bc1b764743888a9e1b81a0d016, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T18:26:28,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/09936caa3c844ff38c534d70b2562fd8 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/09936caa3c844ff38c534d70b2562fd8 2024-11-13T18:26:28,903 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/09936caa3c844ff38c534d70b2562fd8, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T18:26:28,904 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 2bf982671ac78b0bc57c6ef3a6b571eb in 1338ms, sequenceid=306, compaction requested=false 2024-11-13T18:26:28,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:28,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:28,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-11-13T18:26:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-11-13T18:26:28,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-13T18:26:28,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5010 sec 2024-11-13T18:26:28,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 1.5190 sec 2024-11-13T18:26:29,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T18:26:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:29,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:29,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113667d00eea1a443909b0c0835e38f5c98_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522389361/Put/seqid=0 2024-11-13T18:26:29,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:29,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522449415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522449414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522449428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:29,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742438_1614 (size=17534) 2024-11-13T18:26:29,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:29,514 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113667d00eea1a443909b0c0835e38f5c98_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113667d00eea1a443909b0c0835e38f5c98_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:29,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/8ab7da4dce51449d9e9cb52976ad3b42, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:29,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/8ab7da4dce51449d9e9cb52976ad3b42 is 175, key is test_row_0/A:col10/1731522389361/Put/seqid=0 2024-11-13T18:26:29,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522449555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:29,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:29,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522449556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:29,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522449556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:26:29,566 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:29,567 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:26:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-13T18:26:29,569 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:26:29,570 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:26:29,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:26:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:26:29,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742439_1615 (size=48639) 2024-11-13T18:26:29,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/8ab7da4dce51449d9e9cb52976ad3b42 2024-11-13T18:26:29,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/4ee6b6ea7531428eba1a9e9e7ed67350 is 50, key is test_row_0/B:col10/1731522389361/Put/seqid=0 2024-11-13T18:26:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:26:29,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742440_1616 (size=12301) 2024-11-13T18:26:29,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-13T18:26:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:29,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/4ee6b6ea7531428eba1a9e9e7ed67350 2024-11-13T18:26:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:29,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:29,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:29,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/ec7c3c75f1e44676b1b002a96b08d763 is 50, key is test_row_0/C:col10/1731522389361/Put/seqid=0 2024-11-13T18:26:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522449766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522449766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:29,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742441_1617 (size=12301) 2024-11-13T18:26:29,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522449778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:29,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/ec7c3c75f1e44676b1b002a96b08d763 2024-11-13T18:26:29,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/8ab7da4dce51449d9e9cb52976ad3b42 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8ab7da4dce51449d9e9cb52976ad3b42 2024-11-13T18:26:29,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8ab7da4dce51449d9e9cb52976ad3b42, entries=250, sequenceid=328, filesize=47.5 K 2024-11-13T18:26:29,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/4ee6b6ea7531428eba1a9e9e7ed67350 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4ee6b6ea7531428eba1a9e9e7ed67350 2024-11-13T18:26:29,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4ee6b6ea7531428eba1a9e9e7ed67350, entries=150, sequenceid=328, filesize=12.0 K 2024-11-13T18:26:29,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/ec7c3c75f1e44676b1b002a96b08d763 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ec7c3c75f1e44676b1b002a96b08d763 2024-11-13T18:26:29,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ec7c3c75f1e44676b1b002a96b08d763, entries=150, sequenceid=328, filesize=12.0 K 2024-11-13T18:26:29,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2bf982671ac78b0bc57c6ef3a6b571eb in 491ms, sequenceid=328, compaction requested=true 2024-11-13T18:26:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:29,854 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:29,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:29,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:29,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:29,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:29,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:29,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:26:29,855 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:29,863 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:29,863 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:29,864 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/B is initiating minor compaction (all files) 2024-11-13T18:26:29,864 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/A is initiating minor compaction (all files) 2024-11-13T18:26:29,864 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/B in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:29,864 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3355b8190be94e71ae3bfaba1d4c2bf4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/351562bc1b764743888a9e1b81a0d016, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4ee6b6ea7531428eba1a9e9e7ed67350] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=36.6 K 2024-11-13T18:26:29,869 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/A in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:29,869 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/9bbe206a3b424d5d888408b20acb9e5f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aa144ee283b045c58f607bd289fb3ee1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8ab7da4dce51449d9e9cb52976ad3b42] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=109.1 K 2024-11-13T18:26:29,869 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:29,869 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/9bbe206a3b424d5d888408b20acb9e5f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aa144ee283b045c58f607bd289fb3ee1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8ab7da4dce51449d9e9cb52976ad3b42] 2024-11-13T18:26:29,870 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3355b8190be94e71ae3bfaba1d4c2bf4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731522385276 2024-11-13T18:26:29,874 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 351562bc1b764743888a9e1b81a0d016, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731522385956 2024-11-13T18:26:29,874 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bbe206a3b424d5d888408b20acb9e5f, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731522385276 2024-11-13T18:26:29,876 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ee6b6ea7531428eba1a9e9e7ed67350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731522388191 2024-11-13T18:26:29,878 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting aa144ee283b045c58f607bd289fb3ee1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731522385956 2024-11-13T18:26:29,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-13T18:26:29,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:29,882 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:26:29,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:29,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:29,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:29,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:29,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:29,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:29,883 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ab7da4dce51449d9e9cb52976ad3b42, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731522388170 2024-11-13T18:26:29,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b6a70ab20abf4c188d9b6382a4b50866_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522389417/Put/seqid=0 2024-11-13T18:26:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:26:29,931 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#B#compaction#520 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:29,931 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/d3619518ab1043bcb46a5c5be7235977 is 50, key is test_row_0/B:col10/1731522389361/Put/seqid=0 2024-11-13T18:26:30,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742442_1618 (size=12454) 2024-11-13T18:26:30,015 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:30,023 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111356e8e969fc7a42939d0df74cca69f6ae_2bf982671ac78b0bc57c6ef3a6b571eb store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:30,026 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111356e8e969fc7a42939d0df74cca69f6ae_2bf982671ac78b0bc57c6ef3a6b571eb, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:30,026 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111356e8e969fc7a42939d0df74cca69f6ae_2bf982671ac78b0bc57c6ef3a6b571eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:30,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742443_1619 (size=12949) 2024-11-13T18:26:30,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742444_1620 (size=4469) 2024-11-13T18:26:30,083 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#A#compaction#521 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:30,084 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/24ecac6059404028862ac6e9296966d2 is 175, key is test_row_0/A:col10/1731522389361/Put/seqid=0 2024-11-13T18:26:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:30,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:30,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742445_1621 (size=31903) 2024-11-13T18:26:30,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522450187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522450194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522450199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:26:30,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522450308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522450309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522450314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:30,442 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b6a70ab20abf4c188d9b6382a4b50866_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b6a70ab20abf4c188d9b6382a4b50866_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:30,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/ec66a439116b4671840449e4b10f503b, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:30,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/ec66a439116b4671840449e4b10f503b is 175, key is test_row_0/A:col10/1731522389417/Put/seqid=0 2024-11-13T18:26:30,475 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/d3619518ab1043bcb46a5c5be7235977 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/d3619518ab1043bcb46a5c5be7235977 2024-11-13T18:26:30,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742446_1622 (size=31255) 2024-11-13T18:26:30,519 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/B of 2bf982671ac78b0bc57c6ef3a6b571eb into d3619518ab1043bcb46a5c5be7235977(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:30,519 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:30,519 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/B, priority=13, startTime=1731522389855; duration=0sec 2024-11-13T18:26:30,519 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:30,519 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:B 2024-11-13T18:26:30,519 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522450518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,531 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:30,531 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/C is initiating minor compaction (all files) 2024-11-13T18:26:30,531 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/C in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:30,531 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/e15cce9ac1e54072a77be260e682c6c4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/09936caa3c844ff38c534d70b2562fd8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ec7c3c75f1e44676b1b002a96b08d763] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=36.6 K 2024-11-13T18:26:30,531 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting e15cce9ac1e54072a77be260e682c6c4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731522385276 2024-11-13T18:26:30,536 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09936caa3c844ff38c534d70b2562fd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731522385956 2024-11-13T18:26:30,538 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec7c3c75f1e44676b1b002a96b08d763, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731522388191 2024-11-13T18:26:30,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522450530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522450530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,572 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#C#compaction#522 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:30,572 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/24ecac6059404028862ac6e9296966d2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/24ecac6059404028862ac6e9296966d2 2024-11-13T18:26:30,573 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/e13697e3b9d24d31acacdcc916fac347 is 50, key is test_row_0/C:col10/1731522389361/Put/seqid=0 2024-11-13T18:26:30,583 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/A of 2bf982671ac78b0bc57c6ef3a6b571eb into 24ecac6059404028862ac6e9296966d2(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:30,583 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:30,583 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/A, priority=13, startTime=1731522389854; duration=0sec 2024-11-13T18:26:30,583 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:30,583 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:A 2024-11-13T18:26:30,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742447_1623 (size=12949) 2024-11-13T18:26:30,633 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/e13697e3b9d24d31acacdcc916fac347 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/e13697e3b9d24d31acacdcc916fac347 2024-11-13T18:26:30,668 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/C of 2bf982671ac78b0bc57c6ef3a6b571eb into e13697e3b9d24d31acacdcc916fac347(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:30,668 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:30,668 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/C, priority=13, startTime=1731522389855; duration=0sec 2024-11-13T18:26:30,668 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:30,668 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:C 2024-11-13T18:26:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:26:30,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522450840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522450846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522450847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:30,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:30,895 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/ec66a439116b4671840449e4b10f503b 2024-11-13T18:26:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/8c04b09d7bc6480f8cf7524379dadd66 is 50, key is test_row_0/B:col10/1731522389417/Put/seqid=0 2024-11-13T18:26:30,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742448_1624 (size=12301) 2024-11-13T18:26:31,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522451352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:31,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:31,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:31,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:31,355 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/8c04b09d7bc6480f8cf7524379dadd66 2024-11-13T18:26:31,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522451359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:31,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522451368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/396cb42e8acd4f068362ba0d922c5e7e is 50, key is test_row_0/C:col10/1731522389417/Put/seqid=0 2024-11-13T18:26:31,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742449_1625 (size=12301) 2024-11-13T18:26:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:26:31,842 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/396cb42e8acd4f068362ba0d922c5e7e 2024-11-13T18:26:31,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/ec66a439116b4671840449e4b10f503b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ec66a439116b4671840449e4b10f503b 2024-11-13T18:26:31,881 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ec66a439116b4671840449e4b10f503b, entries=150, sequenceid=341, filesize=30.5 K 2024-11-13T18:26:31,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/8c04b09d7bc6480f8cf7524379dadd66 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8c04b09d7bc6480f8cf7524379dadd66 2024-11-13T18:26:31,908 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8c04b09d7bc6480f8cf7524379dadd66, entries=150, sequenceid=341, filesize=12.0 K 2024-11-13T18:26:31,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/396cb42e8acd4f068362ba0d922c5e7e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/396cb42e8acd4f068362ba0d922c5e7e 2024-11-13T18:26:31,954 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/396cb42e8acd4f068362ba0d922c5e7e, entries=150, sequenceid=341, filesize=12.0 K 2024-11-13T18:26:31,964 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2bf982671ac78b0bc57c6ef3a6b571eb in 2082ms, sequenceid=341, compaction requested=false 2024-11-13T18:26:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-13T18:26:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-13T18:26:31,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-13T18:26:31,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4040 sec 2024-11-13T18:26:32,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 2.4360 sec 2024-11-13T18:26:32,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:26:32,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:32,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:32,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:32,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:32,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:32,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522452406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:32,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522452411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:32,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522452418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:32,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113478c053c9fc343838104540057614995_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522392374/Put/seqid=0 2024-11-13T18:26:32,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742450_1626 (size=14994) 2024-11-13T18:26:32,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522452528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:32,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522452528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:32,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522452528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:32,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:32,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522452746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:32,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:32,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522452746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522452746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:32,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:32,915 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113478c053c9fc343838104540057614995_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113478c053c9fc343838104540057614995_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:32,916 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/11896cb9128b4d05940e484b869b0c18, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:32,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/11896cb9128b4d05940e484b869b0c18 is 175, key is test_row_0/A:col10/1731522392374/Put/seqid=0 2024-11-13T18:26:32,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742451_1627 (size=39949) 2024-11-13T18:26:32,975 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/11896cb9128b4d05940e484b869b0c18 2024-11-13T18:26:33,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/fcb86524b4244d5fa4cc8d94e254094c is 50, key is test_row_0/B:col10/1731522392374/Put/seqid=0 2024-11-13T18:26:33,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742452_1628 (size=12301) 2024-11-13T18:26:33,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/fcb86524b4244d5fa4cc8d94e254094c 2024-11-13T18:26:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522453065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522453066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522453067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/d8e5d72d1e4a49b18489b16977595efd is 50, key is test_row_0/C:col10/1731522392374/Put/seqid=0 2024-11-13T18:26:33,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742453_1629 (size=12301) 2024-11-13T18:26:33,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/d8e5d72d1e4a49b18489b16977595efd 2024-11-13T18:26:33,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/11896cb9128b4d05940e484b869b0c18 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/11896cb9128b4d05940e484b869b0c18 2024-11-13T18:26:33,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/11896cb9128b4d05940e484b869b0c18, entries=200, sequenceid=364, filesize=39.0 K 2024-11-13T18:26:33,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/fcb86524b4244d5fa4cc8d94e254094c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/fcb86524b4244d5fa4cc8d94e254094c 2024-11-13T18:26:33,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/fcb86524b4244d5fa4cc8d94e254094c, entries=150, sequenceid=364, filesize=12.0 K 2024-11-13T18:26:33,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/d8e5d72d1e4a49b18489b16977595efd as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d8e5d72d1e4a49b18489b16977595efd 2024-11-13T18:26:33,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d8e5d72d1e4a49b18489b16977595efd, entries=150, sequenceid=364, filesize=12.0 K 2024-11-13T18:26:33,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2bf982671ac78b0bc57c6ef3a6b571eb in 955ms, sequenceid=364, compaction requested=true 2024-11-13T18:26:33,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:33,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:33,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:33,335 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:33,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:33,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:33,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:33,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:26:33,338 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:33,341 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:33,341 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/A is initiating minor compaction (all files) 2024-11-13T18:26:33,341 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/A in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:33,341 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/24ecac6059404028862ac6e9296966d2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ec66a439116b4671840449e4b10f503b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/11896cb9128b4d05940e484b869b0c18] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=100.7 K 2024-11-13T18:26:33,341 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:33,341 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/24ecac6059404028862ac6e9296966d2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ec66a439116b4671840449e4b10f503b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/11896cb9128b4d05940e484b869b0c18] 2024-11-13T18:26:33,342 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24ecac6059404028862ac6e9296966d2, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731522388191 2024-11-13T18:26:33,342 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec66a439116b4671840449e4b10f503b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731522389398 2024-11-13T18:26:33,347 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11896cb9128b4d05940e484b869b0c18, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731522390142 2024-11-13T18:26:33,354 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:33,354 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/B is initiating minor compaction (all files) 2024-11-13T18:26:33,354 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/B in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:33,354 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/d3619518ab1043bcb46a5c5be7235977, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8c04b09d7bc6480f8cf7524379dadd66, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/fcb86524b4244d5fa4cc8d94e254094c] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=36.7 K 2024-11-13T18:26:33,359 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting d3619518ab1043bcb46a5c5be7235977, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731522388191 2024-11-13T18:26:33,363 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c04b09d7bc6480f8cf7524379dadd66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731522389398 2024-11-13T18:26:33,367 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb86524b4244d5fa4cc8d94e254094c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731522390142 2024-11-13T18:26:33,405 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:33,414 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#B#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:33,414 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/f65fb88d0888431d8bd79fb178fb2227 is 50, key is test_row_0/B:col10/1731522392374/Put/seqid=0 2024-11-13T18:26:33,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T18:26:33,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:33,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:33,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:33,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:33,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:33,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:33,421 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113cd617b758d1a4ed0a34e855854b2356e_2bf982671ac78b0bc57c6ef3a6b571eb store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:33,423 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113cd617b758d1a4ed0a34e855854b2356e_2bf982671ac78b0bc57c6ef3a6b571eb, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:33,423 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cd617b758d1a4ed0a34e855854b2356e_2bf982671ac78b0bc57c6ef3a6b571eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:33,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ed204ced55bb4081afc14f804cc8ebef_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522393411/Put/seqid=0 2024-11-13T18:26:33,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742455_1631 (size=4469) 2024-11-13T18:26:33,497 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#A#compaction#528 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:33,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742454_1630 (size=13051) 2024-11-13T18:26:33,499 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/4499e221b91549dc903dd01f8bf2165d is 175, key is test_row_0/A:col10/1731522392374/Put/seqid=0 2024-11-13T18:26:33,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742456_1632 (size=14994) 2024-11-13T18:26:33,519 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:33,526 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ed204ced55bb4081afc14f804cc8ebef_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ed204ced55bb4081afc14f804cc8ebef_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:33,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/aab4056ea4e04edeb7a14a8afb0c3017, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:33,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/aab4056ea4e04edeb7a14a8afb0c3017 is 175, key is test_row_0/A:col10/1731522393411/Put/seqid=0 2024-11-13T18:26:33,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742457_1633 (size=32005) 2024-11-13T18:26:33,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742458_1634 (size=39949) 2024-11-13T18:26:33,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522453573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522453575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522453580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522453589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522453589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522453696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522453705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T18:26:33,811 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:33,815 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:26:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-13T18:26:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T18:26:33,831 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:26:33,832 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:26:33,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:26:33,919 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/f65fb88d0888431d8bd79fb178fb2227 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/f65fb88d0888431d8bd79fb178fb2227 2024-11-13T18:26:33,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522453916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,925 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/B of 2bf982671ac78b0bc57c6ef3a6b571eb into f65fb88d0888431d8bd79fb178fb2227(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:33,925 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:33,925 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/B, priority=13, startTime=1731522393335; duration=0sec 2024-11-13T18:26:33,925 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:33,925 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:B 2024-11-13T18:26:33,925 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:33,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522453925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:33,937 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:33,937 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/C is initiating minor compaction (all files) 2024-11-13T18:26:33,937 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/C in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:33,937 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/e13697e3b9d24d31acacdcc916fac347, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/396cb42e8acd4f068362ba0d922c5e7e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d8e5d72d1e4a49b18489b16977595efd] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=36.7 K 2024-11-13T18:26:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T18:26:33,948 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting e13697e3b9d24d31acacdcc916fac347, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731522388191 2024-11-13T18:26:33,955 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 396cb42e8acd4f068362ba0d922c5e7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731522389398 2024-11-13T18:26:33,962 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting d8e5d72d1e4a49b18489b16977595efd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731522390142 2024-11-13T18:26:33,984 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/4499e221b91549dc903dd01f8bf2165d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/4499e221b91549dc903dd01f8bf2165d 2024-11-13T18:26:33,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T18:26:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:33,985 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#C#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:33,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:33,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:33,986 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/d2213acde0264a2e99f4be4a5394b38d is 50, key is test_row_0/C:col10/1731522392374/Put/seqid=0 2024-11-13T18:26:33,987 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/aab4056ea4e04edeb7a14a8afb0c3017 2024-11-13T18:26:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,003 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/A of 2bf982671ac78b0bc57c6ef3a6b571eb into 4499e221b91549dc903dd01f8bf2165d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:34,003 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:34,003 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/A, priority=13, startTime=1731522393335; duration=0sec 2024-11-13T18:26:34,003 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:34,003 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:A 2024-11-13T18:26:34,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742459_1635 (size=13051) 2024-11-13T18:26:34,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/dbdd8df5a2ed4efd8e40e972007cf1b2 is 50, key is test_row_0/B:col10/1731522393411/Put/seqid=0 2024-11-13T18:26:34,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742460_1636 (size=12301) 2024-11-13T18:26:34,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/dbdd8df5a2ed4efd8e40e972007cf1b2 2024-11-13T18:26:34,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/75a1027a7d52438a848b0408c0d89065 is 50, key is test_row_0/C:col10/1731522393411/Put/seqid=0 2024-11-13T18:26:34,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742461_1637 (size=12301) 2024-11-13T18:26:34,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/75a1027a7d52438a848b0408c0d89065 2024-11-13T18:26:34,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/aab4056ea4e04edeb7a14a8afb0c3017 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aab4056ea4e04edeb7a14a8afb0c3017 2024-11-13T18:26:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T18:26:34,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aab4056ea4e04edeb7a14a8afb0c3017, entries=200, sequenceid=377, filesize=39.0 K 2024-11-13T18:26:34,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T18:26:34,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:34,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/dbdd8df5a2ed4efd8e40e972007cf1b2 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/dbdd8df5a2ed4efd8e40e972007cf1b2 2024-11-13T18:26:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/dbdd8df5a2ed4efd8e40e972007cf1b2, entries=150, sequenceid=377, filesize=12.0 K 2024-11-13T18:26:34,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/75a1027a7d52438a848b0408c0d89065 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/75a1027a7d52438a848b0408c0d89065 2024-11-13T18:26:34,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/75a1027a7d52438a848b0408c0d89065, entries=150, sequenceid=377, filesize=12.0 K 2024-11-13T18:26:34,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2bf982671ac78b0bc57c6ef3a6b571eb in 758ms, sequenceid=377, compaction requested=false 2024-11-13T18:26:34,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(8855): Flush requested on 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:34,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T18:26:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:34,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133805bcde99cd47eaa0e57e944f6abd72_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522394237/Put/seqid=0 2024-11-13T18:26:34,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742462_1638 (size=12454) 2024-11-13T18:26:34,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522454278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522454279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,290 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T18:26:34,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:34,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,322 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133805bcde99cd47eaa0e57e944f6abd72_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133805bcde99cd47eaa0e57e944f6abd72_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:34,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/8498bb42afff470d910557ffea4d347c, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:34,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/8498bb42afff470d910557ffea4d347c is 175, key is test_row_0/A:col10/1731522394237/Put/seqid=0 2024-11-13T18:26:34,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742463_1639 (size=31255) 2024-11-13T18:26:34,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522454399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522454400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T18:26:34,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T18:26:34,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:34,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,479 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/d2213acde0264a2e99f4be4a5394b38d as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2213acde0264a2e99f4be4a5394b38d 2024-11-13T18:26:34,523 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/C of 2bf982671ac78b0bc57c6ef3a6b571eb into d2213acde0264a2e99f4be4a5394b38d(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:34,523 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:34,523 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/C, priority=13, startTime=1731522393335; duration=0sec 2024-11-13T18:26:34,523 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:34,523 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:C 2024-11-13T18:26:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51650 deadline: 1731522454606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,619 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_1' in 2bf982671ac78b0bc57c6ef3a6b571eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522454620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51666 deadline: 1731522454621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522454622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51626 deadline: 1731522454622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,631 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_2' in 2bf982671ac78b0bc57c6ef3a6b571eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,635 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 62ae331128c9,39615,1731522225114 for 'test_row_1' in 2bf982671ac78b0bc57c6ef3a6b571eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T18:26:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T18:26:34,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:34,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,807 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/8498bb42afff470d910557ffea4d347c 2024-11-13T18:26:34,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T18:26:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/23220e17391b453d8b3cb08163fa6e3f is 50, key is test_row_0/B:col10/1731522394237/Put/seqid=0 2024-11-13T18:26:34,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742464_1640 (size=12301) 2024-11-13T18:26:34,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/23220e17391b453d8b3cb08163fa6e3f 2024-11-13T18:26:34,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51592 deadline: 1731522454939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T18:26:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39615 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51606 deadline: 1731522454939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 2024-11-13T18:26:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 , the old value is region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:26:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., hostname=62ae331128c9,39615,1731522225114, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:26:34,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/1342f891645048cea80a5782c45425b7 is 50, key is test_row_0/C:col10/1731522394237/Put/seqid=0 2024-11-13T18:26:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T18:26:34,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T18:26:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. as already flushing 2024-11-13T18:26:34,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:34,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:35,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742465_1641 (size=12301) 2024-11-13T18:26:35,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/1342f891645048cea80a5782c45425b7 2024-11-13T18:26:35,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/8498bb42afff470d910557ffea4d347c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8498bb42afff470d910557ffea4d347c 2024-11-13T18:26:35,053 INFO [Thread-2431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2431. 2024-11-13T18:26:35,053 DEBUG [Thread-2431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:35,053 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,054 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,054 INFO [Thread-2427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2427. 2024-11-13T18:26:35,054 DEBUG [Thread-2427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:35,054 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:35,054 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:35,060 INFO [Thread-2433 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2433. 2024-11-13T18:26:35,061 DEBUG [Thread-2433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:35,061 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,061 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:35,062 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-11-13T18:26:35,062 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:35,062 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,062 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:35,075 INFO [Thread-2425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2425. 2024-11-13T18:26:35,075 DEBUG [Thread-2425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:35,075 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,075 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:35,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8498bb42afff470d910557ffea4d347c, entries=150, sequenceid=399, filesize=30.5 K 2024-11-13T18:26:35,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/23220e17391b453d8b3cb08163fa6e3f as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/23220e17391b453d8b3cb08163fa6e3f 2024-11-13T18:26:35,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/23220e17391b453d8b3cb08163fa6e3f, entries=150, sequenceid=399, filesize=12.0 K 2024-11-13T18:26:35,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/1342f891645048cea80a5782c45425b7 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1342f891645048cea80a5782c45425b7 2024-11-13T18:26:35,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1342f891645048cea80a5782c45425b7, entries=150, sequenceid=399, filesize=12.0 K 2024-11-13T18:26:35,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2bf982671ac78b0bc57c6ef3a6b571eb in 892ms, sequenceid=399, compaction requested=true 2024-11-13T18:26:35,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:35,131 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:35,132 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:35,132 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/A is initiating minor compaction (all files) 2024-11-13T18:26:35,132 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/A in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:35,132 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/4499e221b91549dc903dd01f8bf2165d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aab4056ea4e04edeb7a14a8afb0c3017, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8498bb42afff470d910557ffea4d347c] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=100.8 K 2024-11-13T18:26:35,132 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:35,132 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. files: [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/4499e221b91549dc903dd01f8bf2165d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aab4056ea4e04edeb7a14a8afb0c3017, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8498bb42afff470d910557ffea4d347c] 2024-11-13T18:26:35,132 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4499e221b91549dc903dd01f8bf2165d, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731522390142 2024-11-13T18:26:35,133 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting aab4056ea4e04edeb7a14a8afb0c3017, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731522392401 2024-11-13T18:26:35,133 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8498bb42afff470d910557ffea4d347c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731522393527 2024-11-13T18:26:35,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:26:35,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:35,136 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:35,137 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:35,137 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/B is initiating minor compaction (all files) 2024-11-13T18:26:35,137 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/B in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:35,138 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/f65fb88d0888431d8bd79fb178fb2227, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/dbdd8df5a2ed4efd8e40e972007cf1b2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/23220e17391b453d8b3cb08163fa6e3f] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=36.8 K 2024-11-13T18:26:35,140 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting f65fb88d0888431d8bd79fb178fb2227, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731522390142 2024-11-13T18:26:35,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:26:35,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:35,140 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting dbdd8df5a2ed4efd8e40e972007cf1b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731522392401 2024-11-13T18:26:35,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bf982671ac78b0bc57c6ef3a6b571eb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:26:35,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:35,141 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 23220e17391b453d8b3cb08163fa6e3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731522393527 2024-11-13T18:26:35,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39615 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T18:26:35,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:35,143 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T18:26:35,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:35,143 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:35,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:35,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:35,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:35,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:35,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:35,152 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113fd2caba094e74a4b97e5b35fb11a76d4_2bf982671ac78b0bc57c6ef3a6b571eb store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:35,155 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113fd2caba094e74a4b97e5b35fb11a76d4_2bf982671ac78b0bc57c6ef3a6b571eb, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:35,156 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fd2caba094e74a4b97e5b35fb11a76d4_2bf982671ac78b0bc57c6ef3a6b571eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:35,156 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#B#compaction#538 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:35,156 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/df5403ba3fe742a9b8594bff69d7b73b is 50, key is test_row_0/B:col10/1731522394237/Put/seqid=0 2024-11-13T18:26:35,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ee8aa31eb9784cb6ac24f2e230582776_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522394277/Put/seqid=0 2024-11-13T18:26:35,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742467_1643 (size=4469) 2024-11-13T18:26:35,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742466_1642 (size=13153) 2024-11-13T18:26:35,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742468_1644 (size=12454) 2024-11-13T18:26:35,233 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/df5403ba3fe742a9b8594bff69d7b73b as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/df5403ba3fe742a9b8594bff69d7b73b 2024-11-13T18:26:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:35,238 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ee8aa31eb9784cb6ac24f2e230582776_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ee8aa31eb9784cb6ac24f2e230582776_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:35,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/78dc3a65ea6c4dfab9f0249f4a971796, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:35,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/78dc3a65ea6c4dfab9f0249f4a971796 is 175, key is test_row_0/A:col10/1731522394277/Put/seqid=0 2024-11-13T18:26:35,241 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/B of 2bf982671ac78b0bc57c6ef3a6b571eb into df5403ba3fe742a9b8594bff69d7b73b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:35,241 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:35,241 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/B, priority=13, startTime=1731522395136; duration=0sec 2024-11-13T18:26:35,242 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:26:35,242 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:B 2024-11-13T18:26:35,242 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:26:35,243 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:26:35,243 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1541): 2bf982671ac78b0bc57c6ef3a6b571eb/C is initiating minor compaction (all files) 2024-11-13T18:26:35,243 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bf982671ac78b0bc57c6ef3a6b571eb/C in TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:35,243 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2213acde0264a2e99f4be4a5394b38d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/75a1027a7d52438a848b0408c0d89065, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1342f891645048cea80a5782c45425b7] into tmpdir=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp, totalSize=36.8 K 2024-11-13T18:26:35,244 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting d2213acde0264a2e99f4be4a5394b38d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731522390142 2024-11-13T18:26:35,244 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 75a1027a7d52438a848b0408c0d89065, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731522392401 2024-11-13T18:26:35,244 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] compactions.Compactor(225): Compacting 1342f891645048cea80a5782c45425b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731522393527 2024-11-13T18:26:35,272 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#C#compaction#540 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:35,273 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/5b1f66ae1a594764bd4c4224b6d0c5bf is 50, key is test_row_0/C:col10/1731522394237/Put/seqid=0 2024-11-13T18:26:35,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742469_1645 (size=31255) 2024-11-13T18:26:35,287 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/78dc3a65ea6c4dfab9f0249f4a971796 2024-11-13T18:26:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/639828131fb84544a1c464a864156104 is 50, key is test_row_0/B:col10/1731522394277/Put/seqid=0 2024-11-13T18:26:35,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742470_1646 (size=13153) 2024-11-13T18:26:35,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742471_1647 (size=12301) 2024-11-13T18:26:35,347 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/639828131fb84544a1c464a864156104 2024-11-13T18:26:35,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/c516d3b807504a7993dd85c768d138ab is 50, key is test_row_0/C:col10/1731522394277/Put/seqid=0 2024-11-13T18:26:35,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742472_1648 (size=12301) 2024-11-13T18:26:35,402 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/c516d3b807504a7993dd85c768d138ab 2024-11-13T18:26:35,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/78dc3a65ea6c4dfab9f0249f4a971796 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/78dc3a65ea6c4dfab9f0249f4a971796 2024-11-13T18:26:35,443 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/78dc3a65ea6c4dfab9f0249f4a971796, entries=150, sequenceid=412, filesize=30.5 K 2024-11-13T18:26:35,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/639828131fb84544a1c464a864156104 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/639828131fb84544a1c464a864156104 2024-11-13T18:26:35,449 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/639828131fb84544a1c464a864156104, entries=150, sequenceid=412, filesize=12.0 K 2024-11-13T18:26:35,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/c516d3b807504a7993dd85c768d138ab as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/c516d3b807504a7993dd85c768d138ab 2024-11-13T18:26:35,455 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/c516d3b807504a7993dd85c768d138ab, entries=150, sequenceid=412, filesize=12.0 K 2024-11-13T18:26:35,457 INFO [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 2bf982671ac78b0bc57c6ef3a6b571eb in 314ms, sequenceid=412, compaction requested=false 2024-11-13T18:26:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/62ae331128c9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-13T18:26:35,458 INFO [Thread-2416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2416. 2024-11-13T18:26:35,458 DEBUG [Thread-2416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:35,458 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-13T18:26:35,458 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:35,458 INFO [Thread-2422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2422. 2024-11-13T18:26:35,458 DEBUG [Thread-2422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:35,459 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,459 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:35,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:35,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-13T18:26:35,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-11-13T18:26:35,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.6480 sec 2024-11-13T18:26:35,608 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bf982671ac78b0bc57c6ef3a6b571eb#A#compaction#537 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:26:35,609 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/a30f6d9137964774a03335e7e3574dae is 175, key is test_row_0/A:col10/1731522394237/Put/seqid=0 2024-11-13T18:26:35,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742473_1649 (size=32107) 2024-11-13T18:26:35,683 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/a30f6d9137964774a03335e7e3574dae as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/a30f6d9137964774a03335e7e3574dae 2024-11-13T18:26:35,715 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/A of 2bf982671ac78b0bc57c6ef3a6b571eb into a30f6d9137964774a03335e7e3574dae(size=31.4 K), total size for store is 61.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:35,715 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:35,715 INFO [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/A, priority=13, startTime=1731522395131; duration=0sec 2024-11-13T18:26:35,715 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:35,715 DEBUG [RS:0;62ae331128c9:39615-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:A 2024-11-13T18:26:35,751 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/5b1f66ae1a594764bd4c4224b6d0c5bf as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/5b1f66ae1a594764bd4c4224b6d0c5bf 2024-11-13T18:26:35,806 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bf982671ac78b0bc57c6ef3a6b571eb/C of 2bf982671ac78b0bc57c6ef3a6b571eb into 5b1f66ae1a594764bd4c4224b6d0c5bf(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:35,806 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bf982671ac78b0bc57c6ef3a6b571eb: 2024-11-13T18:26:35,806 INFO [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., storeName=2bf982671ac78b0bc57c6ef3a6b571eb/C, priority=13, startTime=1731522395140; duration=0sec 2024-11-13T18:26:35,806 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:26:35,806 DEBUG [RS:0;62ae331128c9:39615-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bf982671ac78b0bc57c6ef3a6b571eb:C 2024-11-13T18:26:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T18:26:35,976 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:36,647 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-11-13T18:26:36,647 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:36,647 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:36,647 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:36,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:36,654 INFO [Thread-2418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2418. 2024-11-13T18:26:36,655 DEBUG [Thread-2418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:36,655 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:36,655 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:36,655 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:36,664 INFO [Thread-2414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2414. 2024-11-13T18:26:36,664 DEBUG [Thread-2414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:26:36,664 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:36,664 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:36,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1476 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1484 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1459 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1519 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1505 2024-11-13T18:26:36,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:26:36,666 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:26:36,666 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:26:36,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:36,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:36,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:36,668 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:26:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:36,674 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522396674"}]},"ts":"1731522396674"} 2024-11-13T18:26:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T18:26:36,690 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:26:36,690 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:26:36,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:26:36,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, UNASSIGN}] 2024-11-13T18:26:36,696 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, UNASSIGN 2024-11-13T18:26:36,697 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=146 updating hbase:meta row=2bf982671ac78b0bc57c6ef3a6b571eb, regionState=CLOSING, regionLocation=62ae331128c9,39615,1731522225114 2024-11-13T18:26:36,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, UNASSIGN because future has completed 2024-11-13T18:26:36,710 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45483 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=62ae331128c9,39615,1731522225114, table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T18:26:36,711 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:26:36,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114}] 2024-11-13T18:26:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T18:26:36,883 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(122): Close 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:36,883 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:26:36,883 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1722): Closing 2bf982671ac78b0bc57c6ef3a6b571eb, disabling compactions & flushes 2024-11-13T18:26:36,883 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:36,883 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:36,883 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. after waiting 0 ms 2024-11-13T18:26:36,883 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:36,883 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(2902): Flushing 2bf982671ac78b0bc57c6ef3a6b571eb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T18:26:36,883 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=A 2024-11-13T18:26:36,883 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:36,884 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=B 2024-11-13T18:26:36,884 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:36,884 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bf982671ac78b0bc57c6ef3a6b571eb, store=C 2024-11-13T18:26:36,884 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T18:26:36,941 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e4b08e6600ba4e9d99e929fe91393a20_2bf982671ac78b0bc57c6ef3a6b571eb is 50, key is test_row_0/A:col10/1731522395457/Put/seqid=0 2024-11-13T18:26:36,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742474_1650 (size=12454) 2024-11-13T18:26:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T18:26:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T18:26:37,355 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:37,359 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e4b08e6600ba4e9d99e929fe91393a20_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e4b08e6600ba4e9d99e929fe91393a20_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:37,360 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/86c09710004949759f2f4e3857431558, store: [table=TestAcidGuarantees family=A region=2bf982671ac78b0bc57c6ef3a6b571eb] 2024-11-13T18:26:37,361 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/86c09710004949759f2f4e3857431558 is 175, key is test_row_0/A:col10/1731522395457/Put/seqid=0 2024-11-13T18:26:37,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742475_1651 (size=31255) 2024-11-13T18:26:37,785 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/86c09710004949759f2f4e3857431558 2024-11-13T18:26:37,795 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/e233cecc8f18494eb6d5bd80036b7788 is 50, key is test_row_0/B:col10/1731522395457/Put/seqid=0 2024-11-13T18:26:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T18:26:37,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742476_1652 (size=12301) 2024-11-13T18:26:38,246 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/e233cecc8f18494eb6d5bd80036b7788 2024-11-13T18:26:38,283 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/80e3a5e09ed44052898f4abe1df83c8e is 50, key is test_row_0/C:col10/1731522395457/Put/seqid=0 2024-11-13T18:26:38,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742477_1653 (size=12301) 2024-11-13T18:26:38,305 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/80e3a5e09ed44052898f4abe1df83c8e 2024-11-13T18:26:38,335 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/A/86c09710004949759f2f4e3857431558 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/86c09710004949759f2f4e3857431558 2024-11-13T18:26:38,394 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/86c09710004949759f2f4e3857431558, entries=150, sequenceid=423, filesize=30.5 K 2024-11-13T18:26:38,399 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/B/e233cecc8f18494eb6d5bd80036b7788 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/e233cecc8f18494eb6d5bd80036b7788 2024-11-13T18:26:38,436 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/e233cecc8f18494eb6d5bd80036b7788, entries=150, sequenceid=423, filesize=12.0 K 2024-11-13T18:26:38,437 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/.tmp/C/80e3a5e09ed44052898f4abe1df83c8e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/80e3a5e09ed44052898f4abe1df83c8e 2024-11-13T18:26:38,448 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/80e3a5e09ed44052898f4abe1df83c8e, entries=150, sequenceid=423, filesize=12.0 K 2024-11-13T18:26:38,449 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2bf982671ac78b0bc57c6ef3a6b571eb in 1566ms, sequenceid=423, compaction requested=true 2024-11-13T18:26:38,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/c819c406fadd4ec6bf115a456aa66961, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/6a9ddfcd199b4c6db16fe8c597d29720, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/159ae1028782407f98dc0f8457464fc6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ca78f7600f1a4790857718597002a30f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ba81572796c94375b2150f36837cf706, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d420e67dd878462d8ac497ed9806a690, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/f3506b6a4cf1445b809137f9890f375b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/09400d81534340be9d87487fa8dccb54, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/5b26704d94c249e8bac935d3edd4f9d3, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/b817aecd397e4c24ba3f86c1983174b7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d068f78d49ab4dc6850931af927bfde8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/07769939c187457bad5b2a8760e7ceec, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/e50f5bc8775e4bc3962b1cc4e518f856, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/3a8ed835199d4108a5ca8b6465a5acea, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/2feb8f1552154c49bf95e1ecbbf1af88, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/bb8326853d6e47de83e10ef54da54bd7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/9bbe206a3b424d5d888408b20acb9e5f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aa144ee283b045c58f607bd289fb3ee1, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8ab7da4dce51449d9e9cb52976ad3b42, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/24ecac6059404028862ac6e9296966d2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ec66a439116b4671840449e4b10f503b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/11896cb9128b4d05940e484b869b0c18, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/4499e221b91549dc903dd01f8bf2165d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aab4056ea4e04edeb7a14a8afb0c3017, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8498bb42afff470d910557ffea4d347c] to archive 2024-11-13T18:26:38,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:26:38,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/c819c406fadd4ec6bf115a456aa66961 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/c819c406fadd4ec6bf115a456aa66961 2024-11-13T18:26:38,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/6a9ddfcd199b4c6db16fe8c597d29720 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/6a9ddfcd199b4c6db16fe8c597d29720 2024-11-13T18:26:38,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/159ae1028782407f98dc0f8457464fc6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/159ae1028782407f98dc0f8457464fc6 2024-11-13T18:26:38,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ca78f7600f1a4790857718597002a30f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ca78f7600f1a4790857718597002a30f 2024-11-13T18:26:38,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ba81572796c94375b2150f36837cf706 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ba81572796c94375b2150f36837cf706 2024-11-13T18:26:38,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d420e67dd878462d8ac497ed9806a690 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d420e67dd878462d8ac497ed9806a690 2024-11-13T18:26:38,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/f3506b6a4cf1445b809137f9890f375b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/f3506b6a4cf1445b809137f9890f375b 2024-11-13T18:26:38,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/09400d81534340be9d87487fa8dccb54 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/09400d81534340be9d87487fa8dccb54 2024-11-13T18:26:38,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/5b26704d94c249e8bac935d3edd4f9d3 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/5b26704d94c249e8bac935d3edd4f9d3 2024-11-13T18:26:38,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/b817aecd397e4c24ba3f86c1983174b7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/b817aecd397e4c24ba3f86c1983174b7 2024-11-13T18:26:38,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d068f78d49ab4dc6850931af927bfde8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/d068f78d49ab4dc6850931af927bfde8 2024-11-13T18:26:38,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/07769939c187457bad5b2a8760e7ceec to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/07769939c187457bad5b2a8760e7ceec 2024-11-13T18:26:38,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/e50f5bc8775e4bc3962b1cc4e518f856 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/e50f5bc8775e4bc3962b1cc4e518f856 2024-11-13T18:26:38,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/3a8ed835199d4108a5ca8b6465a5acea to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/3a8ed835199d4108a5ca8b6465a5acea 2024-11-13T18:26:38,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/2feb8f1552154c49bf95e1ecbbf1af88 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/2feb8f1552154c49bf95e1ecbbf1af88 2024-11-13T18:26:38,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/bb8326853d6e47de83e10ef54da54bd7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/bb8326853d6e47de83e10ef54da54bd7 2024-11-13T18:26:38,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/9bbe206a3b424d5d888408b20acb9e5f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/9bbe206a3b424d5d888408b20acb9e5f 2024-11-13T18:26:38,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aa144ee283b045c58f607bd289fb3ee1 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aa144ee283b045c58f607bd289fb3ee1 2024-11-13T18:26:38,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8ab7da4dce51449d9e9cb52976ad3b42 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8ab7da4dce51449d9e9cb52976ad3b42 2024-11-13T18:26:38,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/24ecac6059404028862ac6e9296966d2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/24ecac6059404028862ac6e9296966d2 2024-11-13T18:26:38,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ec66a439116b4671840449e4b10f503b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/ec66a439116b4671840449e4b10f503b 2024-11-13T18:26:38,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/11896cb9128b4d05940e484b869b0c18 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/11896cb9128b4d05940e484b869b0c18 2024-11-13T18:26:38,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/4499e221b91549dc903dd01f8bf2165d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/4499e221b91549dc903dd01f8bf2165d 2024-11-13T18:26:38,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aab4056ea4e04edeb7a14a8afb0c3017 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/aab4056ea4e04edeb7a14a8afb0c3017 2024-11-13T18:26:38,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8498bb42afff470d910557ffea4d347c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/8498bb42afff470d910557ffea4d347c 2024-11-13T18:26:38,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/cefa50878a424774bf64903f20259523, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/785275aaad8640dfa931eb95d3692237, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/c40cffcebdaa4edd9b03068f4a9f3e05, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/7abcae5c36c34ec0bbf9e0a2a939ed18, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3a777837916440c99b9d4ec10846f76a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/b4da19b6232941d5ac0e6ed648f9dcb5, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/5e157d97752d493494695512051b2152, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a54051ff0c641659970d683ad988022, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8e98dc3b46194d469647a900f1cf3ca0, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/79e64f779fbe46ac8ef5381a2d83c47e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a10ea47a69b426a964f68f530312af6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4d680e86d0a74050b3528eed55807262, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/6776805ee3314291ae15ee9a31240913, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/90f08e5cdd7e46c0a51f14066d882f06, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4b9f7fe442c8426a811e9551434c41b8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3355b8190be94e71ae3bfaba1d4c2bf4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/db6204de1f2d4c9cad23a88c401bca65, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/351562bc1b764743888a9e1b81a0d016, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/d3619518ab1043bcb46a5c5be7235977, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4ee6b6ea7531428eba1a9e9e7ed67350, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8c04b09d7bc6480f8cf7524379dadd66, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/f65fb88d0888431d8bd79fb178fb2227, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/fcb86524b4244d5fa4cc8d94e254094c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/dbdd8df5a2ed4efd8e40e972007cf1b2, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/23220e17391b453d8b3cb08163fa6e3f] to archive 2024-11-13T18:26:38,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:26:38,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/cefa50878a424774bf64903f20259523 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/cefa50878a424774bf64903f20259523 2024-11-13T18:26:38,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/785275aaad8640dfa931eb95d3692237 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/785275aaad8640dfa931eb95d3692237 2024-11-13T18:26:38,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/c40cffcebdaa4edd9b03068f4a9f3e05 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/c40cffcebdaa4edd9b03068f4a9f3e05 2024-11-13T18:26:38,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/7abcae5c36c34ec0bbf9e0a2a939ed18 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/7abcae5c36c34ec0bbf9e0a2a939ed18 2024-11-13T18:26:38,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3a777837916440c99b9d4ec10846f76a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3a777837916440c99b9d4ec10846f76a 2024-11-13T18:26:38,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/b4da19b6232941d5ac0e6ed648f9dcb5 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/b4da19b6232941d5ac0e6ed648f9dcb5 2024-11-13T18:26:38,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/5e157d97752d493494695512051b2152 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/5e157d97752d493494695512051b2152 2024-11-13T18:26:38,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a54051ff0c641659970d683ad988022 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a54051ff0c641659970d683ad988022 2024-11-13T18:26:38,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8e98dc3b46194d469647a900f1cf3ca0 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8e98dc3b46194d469647a900f1cf3ca0 2024-11-13T18:26:38,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/79e64f779fbe46ac8ef5381a2d83c47e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/79e64f779fbe46ac8ef5381a2d83c47e 2024-11-13T18:26:38,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a10ea47a69b426a964f68f530312af6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/9a10ea47a69b426a964f68f530312af6 2024-11-13T18:26:38,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4d680e86d0a74050b3528eed55807262 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4d680e86d0a74050b3528eed55807262 2024-11-13T18:26:38,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/6776805ee3314291ae15ee9a31240913 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/6776805ee3314291ae15ee9a31240913 2024-11-13T18:26:38,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/90f08e5cdd7e46c0a51f14066d882f06 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/90f08e5cdd7e46c0a51f14066d882f06 2024-11-13T18:26:38,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4b9f7fe442c8426a811e9551434c41b8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4b9f7fe442c8426a811e9551434c41b8 2024-11-13T18:26:38,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3355b8190be94e71ae3bfaba1d4c2bf4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/3355b8190be94e71ae3bfaba1d4c2bf4 2024-11-13T18:26:38,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/db6204de1f2d4c9cad23a88c401bca65 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/db6204de1f2d4c9cad23a88c401bca65 2024-11-13T18:26:38,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/351562bc1b764743888a9e1b81a0d016 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/351562bc1b764743888a9e1b81a0d016 2024-11-13T18:26:38,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/d3619518ab1043bcb46a5c5be7235977 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/d3619518ab1043bcb46a5c5be7235977 2024-11-13T18:26:38,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4ee6b6ea7531428eba1a9e9e7ed67350 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/4ee6b6ea7531428eba1a9e9e7ed67350 2024-11-13T18:26:38,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8c04b09d7bc6480f8cf7524379dadd66 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/8c04b09d7bc6480f8cf7524379dadd66 2024-11-13T18:26:38,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/f65fb88d0888431d8bd79fb178fb2227 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/f65fb88d0888431d8bd79fb178fb2227 2024-11-13T18:26:38,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/fcb86524b4244d5fa4cc8d94e254094c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/fcb86524b4244d5fa4cc8d94e254094c 2024-11-13T18:26:38,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/dbdd8df5a2ed4efd8e40e972007cf1b2 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/dbdd8df5a2ed4efd8e40e972007cf1b2 2024-11-13T18:26:38,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/23220e17391b453d8b3cb08163fa6e3f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/23220e17391b453d8b3cb08163fa6e3f 2024-11-13T18:26:38,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cc8ae2bd1a3445bca9da73d836ec3a9b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/c70a58b6487e42feb1ae5d896139a955, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/691639da75d147ab88ad072a93f3c9f7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2742ae5a6da44a58a42e65f6ee4baff, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/afa84e0279c34ecaba0443dc7776603b, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/911083143bcd43ea901914847ddd864e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/928f3aab379b434cb5afa582987eb57d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1b1c59c55f04428088a3a8e1aa33bb73, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/da9921f948b945df9d20f2c8001ff5d7, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/85bc0588ac79466b86ff5862bec8a7e6, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ff03213ff63c4415b04079c67075e8b9, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/07803082d4984909800009b97e55715a, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/78cb2d1b555c4aad81922b0fdf31ae72, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/933529dd67db4f25ab16e816a7a70f3f, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1a8872294209442da51ccdba08679801, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/e15cce9ac1e54072a77be260e682c6c4, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cd8040353fc346dea33c0d705d0f2e2c, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/09936caa3c844ff38c534d70b2562fd8, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/e13697e3b9d24d31acacdcc916fac347, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ec7c3c75f1e44676b1b002a96b08d763, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/396cb42e8acd4f068362ba0d922c5e7e, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2213acde0264a2e99f4be4a5394b38d, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d8e5d72d1e4a49b18489b16977595efd, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/75a1027a7d52438a848b0408c0d89065, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1342f891645048cea80a5782c45425b7] to archive 2024-11-13T18:26:38,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:26:38,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cc8ae2bd1a3445bca9da73d836ec3a9b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cc8ae2bd1a3445bca9da73d836ec3a9b 2024-11-13T18:26:38,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/c70a58b6487e42feb1ae5d896139a955 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/c70a58b6487e42feb1ae5d896139a955 2024-11-13T18:26:38,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/691639da75d147ab88ad072a93f3c9f7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/691639da75d147ab88ad072a93f3c9f7 2024-11-13T18:26:38,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2742ae5a6da44a58a42e65f6ee4baff to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2742ae5a6da44a58a42e65f6ee4baff 2024-11-13T18:26:38,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/afa84e0279c34ecaba0443dc7776603b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/afa84e0279c34ecaba0443dc7776603b 2024-11-13T18:26:38,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/911083143bcd43ea901914847ddd864e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/911083143bcd43ea901914847ddd864e 2024-11-13T18:26:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T18:26:38,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/928f3aab379b434cb5afa582987eb57d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/928f3aab379b434cb5afa582987eb57d 2024-11-13T18:26:38,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1b1c59c55f04428088a3a8e1aa33bb73 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1b1c59c55f04428088a3a8e1aa33bb73 2024-11-13T18:26:38,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/da9921f948b945df9d20f2c8001ff5d7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/da9921f948b945df9d20f2c8001ff5d7 2024-11-13T18:26:38,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/85bc0588ac79466b86ff5862bec8a7e6 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/85bc0588ac79466b86ff5862bec8a7e6 2024-11-13T18:26:38,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ff03213ff63c4415b04079c67075e8b9 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ff03213ff63c4415b04079c67075e8b9 2024-11-13T18:26:38,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/07803082d4984909800009b97e55715a to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/07803082d4984909800009b97e55715a 2024-11-13T18:26:38,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/78cb2d1b555c4aad81922b0fdf31ae72 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/78cb2d1b555c4aad81922b0fdf31ae72 2024-11-13T18:26:38,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/933529dd67db4f25ab16e816a7a70f3f to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/933529dd67db4f25ab16e816a7a70f3f 2024-11-13T18:26:38,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1a8872294209442da51ccdba08679801 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1a8872294209442da51ccdba08679801 2024-11-13T18:26:38,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/e15cce9ac1e54072a77be260e682c6c4 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/e15cce9ac1e54072a77be260e682c6c4 2024-11-13T18:26:38,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cd8040353fc346dea33c0d705d0f2e2c to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/cd8040353fc346dea33c0d705d0f2e2c 2024-11-13T18:26:38,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/09936caa3c844ff38c534d70b2562fd8 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/09936caa3c844ff38c534d70b2562fd8 2024-11-13T18:26:38,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/e13697e3b9d24d31acacdcc916fac347 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/e13697e3b9d24d31acacdcc916fac347 2024-11-13T18:26:38,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ec7c3c75f1e44676b1b002a96b08d763 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/ec7c3c75f1e44676b1b002a96b08d763 2024-11-13T18:26:38,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/396cb42e8acd4f068362ba0d922c5e7e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/396cb42e8acd4f068362ba0d922c5e7e 2024-11-13T18:26:38,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2213acde0264a2e99f4be4a5394b38d to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d2213acde0264a2e99f4be4a5394b38d 2024-11-13T18:26:38,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d8e5d72d1e4a49b18489b16977595efd to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/d8e5d72d1e4a49b18489b16977595efd 2024-11-13T18:26:38,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/75a1027a7d52438a848b0408c0d89065 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/75a1027a7d52438a848b0408c0d89065 2024-11-13T18:26:38,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1342f891645048cea80a5782c45425b7 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/1342f891645048cea80a5782c45425b7 2024-11-13T18:26:38,930 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/recovered.edits/426.seqid, newMaxSeqId=426, maxSeqId=4 2024-11-13T18:26:38,931 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. 2024-11-13T18:26:38,931 DEBUG [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1676): Region close journal for 2bf982671ac78b0bc57c6ef3a6b571eb: Waiting for close lock at 1731522396883Running coprocessor pre-close hooks at 1731522396883Disabling compacts and flushes for region at 1731522396883Disabling writes for close at 1731522396883Obtaining lock to block concurrent updates at 1731522396883Preparing flush snapshotting stores in 2bf982671ac78b0bc57c6ef3a6b571eb at 1731522396883Finished memstore snapshotting TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731522396884 (+1 ms)Flushing stores of TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb. at 1731522396888 (+4 ms)Flushing 2bf982671ac78b0bc57c6ef3a6b571eb/A: creating writer at 1731522396888Flushing mob file 2bf982671ac78b0bc57c6ef3a6b571eb/A: appending metadata at 1731522396941 (+53 ms)Flushing mob file 2bf982671ac78b0bc57c6ef3a6b571eb/A: closing flushed file at 1731522396941Flushing 2bf982671ac78b0bc57c6ef3a6b571eb/A: appending metadata at 1731522397360 (+419 ms)Flushing 2bf982671ac78b0bc57c6ef3a6b571eb/A: closing flushed file at 1731522397360Flushing 2bf982671ac78b0bc57c6ef3a6b571eb/B: creating writer at 1731522397791 (+431 ms)Flushing 2bf982671ac78b0bc57c6ef3a6b571eb/B: appending metadata at 1731522397794 (+3 ms)Flushing 2bf982671ac78b0bc57c6ef3a6b571eb/B: closing flushed file at 1731522397794Flushing 2bf982671ac78b0bc57c6ef3a6b571eb/C: creating writer at 1731522398262 (+468 ms)Flushing 2bf982671ac78b0bc57c6ef3a6b571eb/C: appending metadata at 1731522398283 (+21 ms)Flushing 2bf982671ac78b0bc57c6ef3a6b571eb/C: closing flushed file at 1731522398283Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@770f16e2: reopening flushed file at 1731522398334 (+51 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66ae22d2: reopening flushed file at 1731522398394 (+60 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37170a34: reopening flushed file at 1731522398436 (+42 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2bf982671ac78b0bc57c6ef3a6b571eb in 1566ms, sequenceid=423, compaction requested=true at 1731522398449 (+13 ms)Writing region close event to WAL at 1731522398927 (+478 ms)Running coprocessor post-close hooks at 1731522398931 (+4 ms)Closed at 1731522398931 2024-11-13T18:26:38,933 INFO [RS_CLOSE_REGION-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(157): Closed 2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:38,935 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=146 updating hbase:meta row=2bf982671ac78b0bc57c6ef3a6b571eb, regionState=CLOSED 2024-11-13T18:26:38,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 because future has completed 2024-11-13T18:26:38,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-13T18:26:38,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; CloseRegionProcedure 2bf982671ac78b0bc57c6ef3a6b571eb, server=62ae331128c9,39615,1731522225114 in 2.2260 sec 2024-11-13T18:26:38,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-13T18:26:38,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bf982671ac78b0bc57c6ef3a6b571eb, UNASSIGN in 2.2450 sec 2024-11-13T18:26:38,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-13T18:26:38,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2500 sec 2024-11-13T18:26:38,959 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522398958"}]},"ts":"1731522398958"} 2024-11-13T18:26:38,962 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:26:38,962 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:26:38,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2950 sec 2024-11-13T18:26:39,699 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:26:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T18:26:40,863 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:40,864 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:26:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:40,871 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:26:40,876 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=148, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:40,891 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:40,895 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C, FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/recovered.edits] 2024-11-13T18:26:40,906 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/78dc3a65ea6c4dfab9f0249f4a971796 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/78dc3a65ea6c4dfab9f0249f4a971796 2024-11-13T18:26:40,927 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/86c09710004949759f2f4e3857431558 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/86c09710004949759f2f4e3857431558 2024-11-13T18:26:40,936 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/a30f6d9137964774a03335e7e3574dae to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/A/a30f6d9137964774a03335e7e3574dae 2024-11-13T18:26:40,957 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/639828131fb84544a1c464a864156104 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/639828131fb84544a1c464a864156104 2024-11-13T18:26:40,967 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/df5403ba3fe742a9b8594bff69d7b73b to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/df5403ba3fe742a9b8594bff69d7b73b 2024-11-13T18:26:40,983 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/e233cecc8f18494eb6d5bd80036b7788 to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/B/e233cecc8f18494eb6d5bd80036b7788 2024-11-13T18:26:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:26:41,015 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/5b1f66ae1a594764bd4c4224b6d0c5bf to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/5b1f66ae1a594764bd4c4224b6d0c5bf 2024-11-13T18:26:41,033 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/80e3a5e09ed44052898f4abe1df83c8e to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/80e3a5e09ed44052898f4abe1df83c8e 2024-11-13T18:26:41,038 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/c516d3b807504a7993dd85c768d138ab to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/C/c516d3b807504a7993dd85c768d138ab 2024-11-13T18:26:41,046 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/recovered.edits/426.seqid to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb/recovered.edits/426.seqid 2024-11-13T18:26:41,046 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/default/TestAcidGuarantees/2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,050 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:26:41,054 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:26:41,064 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T18:26:41,122 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111312962d76972c4790a6c0f71400c9fe27_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111312962d76972c4790a6c0f71400c9fe27_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,142 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c5f1d19c7f141c08260149a09ef894c_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c5f1d19c7f141c08260149a09ef894c_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111324e047ae96f94cd28cf017390b212bd6_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111324e047ae96f94cd28cf017390b212bd6_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,183 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c2f2c80e04b47b78d23c1bbdb446112_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c2f2c80e04b47b78d23c1bbdb446112_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:26:41,197 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132d35ed8e39db4940831973bbee533fa7_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132d35ed8e39db4940831973bbee533fa7_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,218 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133805bcde99cd47eaa0e57e944f6abd72_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133805bcde99cd47eaa0e57e944f6abd72_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,224 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113478c053c9fc343838104540057614995_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113478c053c9fc343838104540057614995_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,226 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134b36556aa20a419b81de0b77f3f0c019_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134b36556aa20a419b81de0b77f3f0c019_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,228 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113667d00eea1a443909b0c0835e38f5c98_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113667d00eea1a443909b0c0835e38f5c98_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,240 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111368b57d3e462249cfb48ed703cbc689d3_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111368b57d3e462249cfb48ed703cbc689d3_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,251 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136ea5da40c95043ea84a7ff77ef935c5a_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136ea5da40c95043ea84a7ff77ef935c5a_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,253 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a2ee873c96414e1793d4b3b5b9be53ee_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a2ee873c96414e1793d4b3b5b9be53ee_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,255 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ab2a7e975ff64f738bde9de966e1a09a_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ab2a7e975ff64f738bde9de966e1a09a_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,261 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b6a70ab20abf4c188d9b6382a4b50866_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b6a70ab20abf4c188d9b6382a4b50866_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,263 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b6e36e8486f648bdb970fc98570bf79a_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b6e36e8486f648bdb970fc98570bf79a_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,264 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e4b08e6600ba4e9d99e929fe91393a20_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e4b08e6600ba4e9d99e929fe91393a20_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,265 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ed204ced55bb4081afc14f804cc8ebef_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ed204ced55bb4081afc14f804cc8ebef_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,267 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ee25f6dd2e554aa0be4aef3530221aa0_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ee25f6dd2e554aa0be4aef3530221aa0_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,268 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ee8aa31eb9784cb6ac24f2e230582776_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ee8aa31eb9784cb6ac24f2e230582776_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,269 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f6e03a0b81264260b71cbfe11c159451_2bf982671ac78b0bc57c6ef3a6b571eb to hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f6e03a0b81264260b71cbfe11c159451_2bf982671ac78b0bc57c6ef3a6b571eb 2024-11-13T18:26:41,270 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:26:41,279 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=148, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:41,289 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:26:41,296 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:26:41,308 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=148, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:41,308 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:26:41,308 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731522401308"}]},"ts":"9223372036854775807"} 2024-11-13T18:26:41,318 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:26:41,318 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2bf982671ac78b0bc57c6ef3a6b571eb, NAME => 'TestAcidGuarantees,,1731522373309.2bf982671ac78b0bc57c6ef3a6b571eb.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:26:41,319 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:26:41,319 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731522401319"}]},"ts":"9223372036854775807"} 2024-11-13T18:26:41,328 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:26:41,330 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=148, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:26:41,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 466 msec 2024-11-13T18:26:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45483 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:26:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:26:41,507 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:26:41,538 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249 (was 251), OpenFileDescriptor=455 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1942 (was 1913) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2923 (was 1777) - AvailableMemoryMB LEAK? - 2024-11-13T18:26:41,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T18:26:41,539 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:26:41,539 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:26:41,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:41,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:41,540 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T18:26:41,540 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1568360069, stopped=false 2024-11-13T18:26:41,540 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=62ae331128c9,45483,1731522221514 2024-11-13T18:26:41,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:41,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:26:41,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:26:41,552 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:26:41,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:26:41,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:26:41,558 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:26:41,558 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:26:41,563 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:26:41,563 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(878): Closing user regions 2024-11-13T18:26:41,563 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:26:41,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:41,576 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '62ae331128c9,39615,1731522225114' ***** 2024-11-13T18:26:41,576 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:26:41,579 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:26:41,579 INFO [RS:0;62ae331128c9:39615 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:26:41,579 INFO [RS:0;62ae331128c9:39615 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:26:41,579 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(959): stopping server 62ae331128c9,39615,1731522225114 2024-11-13T18:26:41,579 INFO [RS:0;62ae331128c9:39615 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:26:41,579 INFO [RS:0;62ae331128c9:39615 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;62ae331128c9:39615. 2024-11-13T18:26:41,579 DEBUG [RS:0;62ae331128c9:39615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:26:41,579 DEBUG [RS:0;62ae331128c9:39615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:41,580 INFO [RS:0;62ae331128c9:39615 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:26:41,580 INFO [RS:0;62ae331128c9:39615 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:26:41,580 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:26:41,580 INFO [RS:0;62ae331128c9:39615 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:26:41,580 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T18:26:41,582 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T18:26:41,582 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T18:26:41,582 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:41,583 INFO [regionserver/62ae331128c9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:26:41,586 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:26:41,586 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:26:41,586 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:26:41,586 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:26:41,586 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:26:41,586 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-13T18:26:41,667 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/info/80c047e1a23b407184278071c2f79b00 is 95, key is TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905./info:/1731522263237/DeleteFamily/seqid=0 2024-11-13T18:26:41,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742478_1654 (size=6514) 2024-11-13T18:26:41,724 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/info/80c047e1a23b407184278071c2f79b00 2024-11-13T18:26:41,782 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:41,819 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/ns/5cf14a3df03048f49c9f640e90941095 is 93, key is TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905./ns:/1731522263114/DeleteFamily/seqid=0 2024-11-13T18:26:41,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742479_1655 (size=5906) 2024-11-13T18:26:41,928 INFO [regionserver/62ae331128c9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T18:26:41,928 INFO [regionserver/62ae331128c9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T18:26:41,983 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:42,183 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:42,241 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/ns/5cf14a3df03048f49c9f640e90941095 2024-11-13T18:26:42,314 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/rep_barrier/b6124d1464a145a7b36152f137c6c875 is 102, key is TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905./rep_barrier:/1731522263114/DeleteFamily/seqid=0 2024-11-13T18:26:42,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742480_1656 (size=6025) 2024-11-13T18:26:42,385 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:42,586 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:42,743 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/rep_barrier/b6124d1464a145a7b36152f137c6c875 2024-11-13T18:26:42,787 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:42,833 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/table/ae03c8e84036487ba45bda7e26ac437a is 96, key is TestAcidGuarantees,,1731522234732.debb09c76feaf68052a52c8303bbb905./table:/1731522263114/DeleteFamily/seqid=0 2024-11-13T18:26:42,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742481_1657 (size=5936) 2024-11-13T18:26:42,990 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:43,194 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:43,269 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/table/ae03c8e84036487ba45bda7e26ac437a 2024-11-13T18:26:43,278 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/info/80c047e1a23b407184278071c2f79b00 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/info/80c047e1a23b407184278071c2f79b00 2024-11-13T18:26:43,326 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/info/80c047e1a23b407184278071c2f79b00, entries=12, sequenceid=90, filesize=6.4 K 2024-11-13T18:26:43,342 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/ns/5cf14a3df03048f49c9f640e90941095 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/ns/5cf14a3df03048f49c9f640e90941095 2024-11-13T18:26:43,398 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:43,419 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/ns/5cf14a3df03048f49c9f640e90941095, entries=8, sequenceid=90, filesize=5.8 K 2024-11-13T18:26:43,439 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/rep_barrier/b6124d1464a145a7b36152f137c6c875 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/rep_barrier/b6124d1464a145a7b36152f137c6c875 2024-11-13T18:26:43,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:26:43,497 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/rep_barrier/b6124d1464a145a7b36152f137c6c875, entries=6, sequenceid=90, filesize=5.9 K 2024-11-13T18:26:43,513 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/.tmp/table/ae03c8e84036487ba45bda7e26ac437a as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/table/ae03c8e84036487ba45bda7e26ac437a 2024-11-13T18:26:43,561 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/table/ae03c8e84036487ba45bda7e26ac437a, entries=7, sequenceid=90, filesize=5.8 K 2024-11-13T18:26:43,571 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1985ms, sequenceid=90, compaction requested=false 2024-11-13T18:26:43,601 DEBUG [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:26:43,646 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-13T18:26:43,655 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:26:43,655 INFO [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:26:43,655 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731522401585Running coprocessor pre-close hooks at 1731522401586 (+1 ms)Disabling compacts and flushes for region at 1731522401586Disabling writes for close at 1731522401586Obtaining lock to block concurrent updates at 1731522401586Preparing flush snapshotting stores in 1588230740 at 1731522401586Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731522401587 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731522401602 (+15 ms)Flushing 1588230740/info: creating writer at 1731522401603 (+1 ms)Flushing 1588230740/info: appending metadata at 1731522401666 (+63 ms)Flushing 1588230740/info: closing flushed file at 1731522401666Flushing 1588230740/ns: creating writer at 1731522401762 (+96 ms)Flushing 1588230740/ns: appending metadata at 1731522401819 (+57 ms)Flushing 1588230740/ns: closing flushed file at 1731522401819Flushing 1588230740/rep_barrier: creating writer at 1731522402274 (+455 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731522402314 (+40 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731522402314Flushing 1588230740/table: creating writer at 1731522402773 (+459 ms)Flushing 1588230740/table: appending metadata at 1731522402832 (+59 ms)Flushing 1588230740/table: closing flushed file at 1731522402832Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b21d5e4: reopening flushed file at 1731522403278 (+446 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36e219f1: reopening flushed file at 1731522403331 (+53 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60689475: reopening flushed file at 1731522403419 (+88 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9265aeb: reopening flushed file at 1731522403498 (+79 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1985ms, sequenceid=90, compaction requested=false at 1731522403571 (+73 ms)Writing region close event to WAL at 1731522403610 (+39 ms)Running coprocessor post-close hooks at 1731522403654 (+44 ms)Closed at 1731522403655 (+1 ms) 2024-11-13T18:26:43,655 DEBUG [RS_CLOSE_META-regionserver/62ae331128c9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T18:26:43,801 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(976): stopping server 62ae331128c9,39615,1731522225114; all regions closed. 2024-11-13T18:26:43,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741834_1010 (size=25344) 2024-11-13T18:26:43,841 DEBUG [RS:0;62ae331128c9:39615 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/oldWALs 2024-11-13T18:26:43,842 INFO [RS:0;62ae331128c9:39615 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 62ae331128c9%2C39615%2C1731522225114.meta:.meta(num 1731522233080) 2024-11-13T18:26:43,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741833_1009 (size=18608853) 2024-11-13T18:26:43,873 DEBUG [RS:0;62ae331128c9:39615 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/oldWALs 2024-11-13T18:26:43,873 INFO [RS:0;62ae331128c9:39615 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 62ae331128c9%2C39615%2C1731522225114:(num 1731522232248) 2024-11-13T18:26:43,873 DEBUG [RS:0;62ae331128c9:39615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:43,873 INFO [RS:0;62ae331128c9:39615 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:26:43,873 INFO [RS:0;62ae331128c9:39615 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:26:43,873 INFO [RS:0;62ae331128c9:39615 {}] hbase.ChoreService(370): Chore service for: regionserver/62ae331128c9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T18:26:43,874 INFO [RS:0;62ae331128c9:39615 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:26:43,874 INFO [regionserver/62ae331128c9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:26:43,904 INFO [RS:0;62ae331128c9:39615 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39615 2024-11-13T18:26:43,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/62ae331128c9,39615,1731522225114 2024-11-13T18:26:43,935 INFO [RS:0;62ae331128c9:39615 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:26:43,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:26:43,947 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [62ae331128c9,39615,1731522225114] 2024-11-13T18:26:43,963 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/62ae331128c9,39615,1731522225114 already deleted, retry=false 2024-11-13T18:26:43,963 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 62ae331128c9,39615,1731522225114 expired; onlineServers=0 2024-11-13T18:26:43,963 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '62ae331128c9,45483,1731522221514' ***** 2024-11-13T18:26:43,963 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T18:26:43,966 INFO [M:0;62ae331128c9:45483 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:26:43,967 INFO [M:0;62ae331128c9:45483 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:26:43,974 DEBUG [M:0;62ae331128c9:45483 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T18:26:43,975 DEBUG [M:0;62ae331128c9:45483 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T18:26:43,975 DEBUG [master/62ae331128c9:0:becomeActiveMaster-HFileCleaner.small.0-1731522231412 {}] cleaner.HFileCleaner(306): Exit Thread[master/62ae331128c9:0:becomeActiveMaster-HFileCleaner.small.0-1731522231412,5,FailOnTimeoutGroup] 2024-11-13T18:26:43,975 INFO [M:0;62ae331128c9:45483 {}] hbase.ChoreService(370): Chore service for: master/62ae331128c9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T18:26:43,975 INFO [M:0;62ae331128c9:45483 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:26:43,975 DEBUG [M:0;62ae331128c9:45483 {}] master.HMaster(1795): Stopping service threads 2024-11-13T18:26:43,975 INFO [M:0;62ae331128c9:45483 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T18:26:43,975 INFO [M:0;62ae331128c9:45483 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:26:43,975 ERROR [M:0;62ae331128c9:45483 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-13T18:26:43,976 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T18:26:43,977 INFO [M:0;62ae331128c9:45483 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:26:43,977 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T18:26:43,978 DEBUG [master/62ae331128c9:0:becomeActiveMaster-HFileCleaner.large.0-1731522231401 {}] cleaner.HFileCleaner(306): Exit Thread[master/62ae331128c9:0:becomeActiveMaster-HFileCleaner.large.0-1731522231401,5,FailOnTimeoutGroup] 2024-11-13T18:26:43,999 DEBUG [M:0;62ae331128c9:45483 {}] zookeeper.ZKUtil(347): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T18:26:43,999 WARN [M:0;62ae331128c9:45483 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T18:26:44,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T18:26:44,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:26:44,003 INFO [M:0;62ae331128c9:45483 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/.lastflushedseqids 2024-11-13T18:26:44,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742482_1658 (size=72) 2024-11-13T18:26:44,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:26:44,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39615-0x101369e15260001, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:26:44,049 INFO [RS:0;62ae331128c9:39615 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:26:44,049 INFO [RS:0;62ae331128c9:39615 {}] regionserver.HRegionServer(1031): Exiting; stopping=62ae331128c9,39615,1731522225114; zookeeper connection closed. 2024-11-13T18:26:44,049 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@26fda55a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@26fda55a 2024-11-13T18:26:44,051 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T18:26:44,439 INFO [M:0;62ae331128c9:45483 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:26:44,443 INFO [M:0;62ae331128c9:45483 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T18:26:44,443 DEBUG [M:0;62ae331128c9:45483 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:26:44,444 INFO [M:0;62ae331128c9:45483 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:26:44,444 DEBUG [M:0;62ae331128c9:45483 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:26:44,444 DEBUG [M:0;62ae331128c9:45483 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:26:44,444 DEBUG [M:0;62ae331128c9:45483 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:26:44,444 INFO [M:0;62ae331128c9:45483 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=756.77 KB heapSize=926.24 KB 2024-11-13T18:26:44,463 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:26:44,508 DEBUG [M:0;62ae331128c9:45483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5640644efb774c27860fe1ce6afdbd9c is 82, key is hbase:meta,,1/info:regioninfo/1731522233497/Put/seqid=0 2024-11-13T18:26:44,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742483_1659 (size=5672) 2024-11-13T18:26:44,547 INFO [M:0;62ae331128c9:45483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2106 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5640644efb774c27860fe1ce6afdbd9c 2024-11-13T18:26:44,583 DEBUG [M:0;62ae331128c9:45483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1271c6d8a9df449484d6a7674246d78e is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x81/proc:d/1731522374789/Put/seqid=0 2024-11-13T18:26:44,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742484_1660 (size=41524) 2024-11-13T18:26:44,616 INFO [M:0;62ae331128c9:45483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=756.22 KB at sequenceid=2106 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1271c6d8a9df449484d6a7674246d78e 2024-11-13T18:26:44,627 INFO [M:0;62ae331128c9:45483 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1271c6d8a9df449484d6a7674246d78e 2024-11-13T18:26:44,691 DEBUG [M:0;62ae331128c9:45483 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e6137707b42f47849a4ae79811fae495 is 69, key is 62ae331128c9,39615,1731522225114/rs:state/1731522231483/Put/seqid=0 2024-11-13T18:26:44,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073742485_1661 (size=5156) 2024-11-13T18:26:44,721 INFO [M:0;62ae331128c9:45483 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2106 (bloomFilter=true), to=hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e6137707b42f47849a4ae79811fae495 2024-11-13T18:26:44,750 DEBUG [M:0;62ae331128c9:45483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5640644efb774c27860fe1ce6afdbd9c as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5640644efb774c27860fe1ce6afdbd9c 2024-11-13T18:26:44,760 INFO [M:0;62ae331128c9:45483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5640644efb774c27860fe1ce6afdbd9c, entries=8, sequenceid=2106, filesize=5.5 K 2024-11-13T18:26:44,786 DEBUG [M:0;62ae331128c9:45483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1271c6d8a9df449484d6a7674246d78e as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1271c6d8a9df449484d6a7674246d78e 2024-11-13T18:26:44,829 INFO [M:0;62ae331128c9:45483 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1271c6d8a9df449484d6a7674246d78e 2024-11-13T18:26:44,829 INFO [M:0;62ae331128c9:45483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1271c6d8a9df449484d6a7674246d78e, entries=148, sequenceid=2106, filesize=40.6 K 2024-11-13T18:26:44,844 DEBUG [M:0;62ae331128c9:45483 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e6137707b42f47849a4ae79811fae495 as hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e6137707b42f47849a4ae79811fae495 2024-11-13T18:26:44,859 INFO [M:0;62ae331128c9:45483 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33321/user/jenkins/test-data/181331b3-101a-45c8-4f4e-07c26775800b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e6137707b42f47849a4ae79811fae495, entries=1, sequenceid=2106, filesize=5.0 K 2024-11-13T18:26:44,863 INFO [M:0;62ae331128c9:45483 {}] regionserver.HRegion(3140): Finished flush of dataSize ~756.77 KB/774936, heapSize ~925.95 KB/948168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 419ms, sequenceid=2106, compaction requested=false 2024-11-13T18:26:44,894 INFO [M:0;62ae331128c9:45483 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:26:44,895 DEBUG [M:0;62ae331128c9:45483 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731522404443Disabling compacts and flushes for region at 1731522404443Disabling writes for close at 1731522404444 (+1 ms)Obtaining lock to block concurrent updates at 1731522404444Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731522404444Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=774936, getHeapSize=948408, getOffHeapSize=0, getCellsCount=2279 at 1731522404444Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731522404445 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731522404445Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731522404508 (+63 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731522404508Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731522404551 (+43 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731522404582 (+31 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731522404582Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731522404627 (+45 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731522404690 (+63 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731522404690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b13ac5a: reopening flushed file at 1731522404745 (+55 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71f1a558: reopening flushed file at 1731522404761 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16fd6341: reopening flushed file at 1731522404829 (+68 ms)Finished flush of dataSize ~756.77 KB/774936, heapSize ~925.95 KB/948168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 419ms, sequenceid=2106, compaction requested=false at 1731522404863 (+34 ms)Writing region close event to WAL at 1731522404894 (+31 ms)Closed at 1731522404894 2024-11-13T18:26:44,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46535 is added to blk_1073741830_1006 (size=912937) 2024-11-13T18:26:44,927 INFO [M:0;62ae331128c9:45483 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T18:26:44,927 INFO [M:0;62ae331128c9:45483 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45483 2024-11-13T18:26:44,927 INFO [M:0;62ae331128c9:45483 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:26:44,927 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:26:45,037 INFO [M:0;62ae331128c9:45483 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:26:45,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:26:45,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45483-0x101369e15260000, quorum=127.0.0.1:63431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:26:45,048 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1e531827{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:26:45,052 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4ea205b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:26:45,052 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:26:45,052 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24457f8e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:26:45,052 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@350067e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/hadoop.log.dir/,STOPPED} 2024-11-13T18:26:45,067 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T18:26:45,067 WARN [BP-1851317081-172.17.0.3-1731522213332 heartbeating to localhost/127.0.0.1:33321 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T18:26:45,067 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T18:26:45,067 WARN [BP-1851317081-172.17.0.3-1731522213332 heartbeating to localhost/127.0.0.1:33321 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1851317081-172.17.0.3-1731522213332 (Datanode Uuid 5776e202-595b-4c76-93e7-e9f26f4804af) service to localhost/127.0.0.1:33321 2024-11-13T18:26:45,072 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/cluster_289219f1-227d-9b3f-386e-b9c909da3090/data/data1/current/BP-1851317081-172.17.0.3-1731522213332 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:26:45,072 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T18:26:45,074 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/cluster_289219f1-227d-9b3f-386e-b9c909da3090/data/data2/current/BP-1851317081-172.17.0.3-1731522213332 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:26:45,099 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f405bdd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:26:45,100 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:26:45,100 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:26:45,101 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:26:45,101 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8cb1606a-b431-5999-6a4b-ec3d158f26a7/hadoop.log.dir/,STOPPED} 2024-11-13T18:26:45,140 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T18:26:45,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down